Individual camera calibration goes well, but stereo calibration rmse is huge

cpm222
Mon, 01 Apr 2024 21:33:01 +0000
Python

I am trying to set up a stereo vision system with some cameras I bought off of arducam, however, I am having issues calibrating the system. I am able to successfully calibrate the individual cameras and I get a RMSE < 0.3, however, when I use stereoCalibrate I get RMSE > 30. Unsurprisingly, when I use these parameters to rectify an image the resulting image is extremely zoomed in, and my depth images are not usable. I calibrate the system on a set of 48 images, and the image size is 1280x720. I have checked every pair of images, and the quality/chessboard generated all look right. I will attach the code I am using as well as an example of one of the images I used to calibrate. The picture I uploaded shows the right and left camera image as well as the corners cv2.findChessboardCorners found. If anyone can suggest some tips that may help me get some decent results out of this I will be extremely appreciative. Thank you!

import glob
import os
import random
import sys

import numpy as np
import cv2

CHESSBOARD_SIZE = (8, 6)
CHESSBOARD_OPTIONS = (cv2.CALIB_CB_ADAPTIVE_THRESH |
        cv2.CALIB_CB_NORMALIZE_IMAGE | cv2.CALIB_CB_FAST_CHECK)

OBJECT_POINT_ZERO = np.zeros((CHESSBOARD_SIZE[0] * CHESSBOARD_SIZE[1], 3),
        np.float32)
OBJECT_POINT_ZERO[:, :2] = np.mgrid[0:CHESSBOARD_SIZE[0],
        0:CHESSBOARD_SIZE[1]].T.reshape(-1, 2)

OPTIMIZE_ALPHA = 0.25

TERMINATION_CRITERIA = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_MAX_ITER, 30,
        0.001)

MAX_IMAGES = 64

temp = 'capture/temp/'
leftImageDir = 'capture/left/'
rightImageDir = 'capture/right'
outputFile = 'camera_params/params.npz'

def readImagesAndFindChessboards(imageDirectory):
    cacheFile = "{0}/chessboards.npz".format(imageDirectory)
    try:
        cache = np.load(cacheFile)
        print("Loading image data from cache file at {0}".format(cacheFile))
        return (list(cache["filenames"]), list(cache["objectPoints"]),
                list(cache["imagePoints"]), tuple(cache["imageSize"]),None)
    except IOError:
        print("Cache file at {0} not found".format(cacheFile))

    print("Reading images at {0}".format(imageDirectory))
    imagePaths = glob.glob("{0}/*.jpg".format(imageDirectory))

    filenames = []
    objectPoints = []
    imagePoints = []
    imageSize = None
    images = []

    for imagePath in sorted(imagePaths):
        image = cv2.imread(imagePath)
        grayImage = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

        newSize = grayImage.shape[::-1]
        if imageSize != None and newSize != imageSize:
            raise ValueError(
                    "Calibration image at {0} is not the same size as the others"
                    .format(imagePath))
        imageSize = newSize

        hasCorners, corners = cv2.findChessboardCorners(grayImage,
                CHESSBOARD_SIZE, flags=cv2.CALIB_CB_FAST_CHECK)

        if hasCorners:
            filenames.append(os.path.basename(imagePath))
            objectPoints.append(OBJECT_POINT_ZERO)
            cv2.cornerSubPix(grayImage, corners, (11, 11), (-1, -1),
                    TERMINATION_CRITERIA)
            imagePoints.append(corners)

        cv2.drawChessboardCorners(image, CHESSBOARD_SIZE, corners, hasCorners)
        #cv2.imshow(imageDirectory, image)
        if hasCorners:
            images.append(image)

        # Needed to draw the window
        #cv2.waitKey(1)

    #cv2.destroyWindow(imageDirectory)

    print("Found corners in {0} out of {1} images"
            .format(len(imagePoints), len(imagePaths)))

    np.savez_compressed(cacheFile,
            filenames=filenames, objectPoints=objectPoints,
            imagePoints=imagePoints, imageSize=imageSize)

    return filenames, objectPoints, imagePoints, imageSize, images,

def compareBoardImages(rightImages,rightFilenames,leftImages,leftFilenames,dir):
    for i in range(len(leftImages)):
        combinedImage = np.concatenate((rightImages[i], leftImages[i]), axis=1)
        cv2.imwrite(dir+rightFilenames[i],combinedImage)


(leftFilenames, leftObjectPoints, leftImagePoints, leftSize, leftImages,
        ) = readImagesAndFindChessboards(leftImageDir)
(rightFilenames, rightObjectPoints, rightImagePoints, rightSize, rightImages,
        ) = readImagesAndFindChessboards(rightImageDir)

if leftImages is not None and rightImages is not None:
    compareBoardImages(rightImages,rightFilenames,leftImages,leftFilenames,temp)

if leftSize != rightSize:
    print("Camera resolutions do not match")
    sys.exit(1)
imageSize = leftSize

filenames = list(set(leftFilenames) & set(rightFilenames))
if (len(filenames) > MAX_IMAGES):
    print("Too many images to calibrate, using {0} randomly selected images"
            .format(MAX_IMAGES))
    filenames = random.sample(filenames, MAX_IMAGES)
filenames = sorted(filenames)
print("Using these images:")
print(filenames)

def getMatchingObjectAndImagePoints(requestedFilenames,
        allFilenames, objectPoints, imagePoints,):
    requestedFilenameSet = set(requestedFilenames)
    requestedObjectPoints = []
    requestedImagePoints = []

    for index, filename in enumerate(allFilenames):
        if filename in requestedFilenameSet:
            requestedObjectPoints.append(objectPoints[index])
            requestedImagePoints.append(imagePoints[index])

    return requestedObjectPoints, requestedImagePoints

leftObjectPoints, leftImagePoints = getMatchingObjectAndImagePoints(filenames,
        leftFilenames, leftObjectPoints, leftImagePoints)
rightObjectPoints, rightImagePoints = getMatchingObjectAndImagePoints(filenames,
        rightFilenames, rightObjectPoints, rightImagePoints)

# TODO: Fix this validation
# Keep getting "Use a.any() or a.all()" even though it's already used?!
# if (leftObjectPoints != rightObjectPoints).all():
#     print("Object points do not match")
#     sys.exit(1)
objectPoints = leftObjectPoints

print("Calibrating left camera...")
rmse, leftCameraMatrix, leftDistortionCoefficients, _, _ = cv2.calibrateCamera(
        objectPoints, leftImagePoints, imageSize, None, None)
print('left error:',rmse)

objectPoints = rightObjectPoints
print("Calibrating right camera...")
rmse, rightCameraMatrix, rightDistortionCoefficients, _, _ = cv2.calibrateCamera(
        objectPoints, rightImagePoints, imageSize, None, None)
print('right error:',rmse)

print("Calibrating cameras together...")
(rmse, _, _, _, _, rotationMatrix, translationVector, _, _) = cv2.stereoCalibrate(
        objectPoints, leftImagePoints, rightImagePoints,
        leftCameraMatrix, leftDistortionCoefficients,
        rightCameraMatrix, rightDistortionCoefficients,
        imageSize, None, None, None, None,
        flags=cv2.CALIB_FIX_INTRINSIC, criteria=TERMINATION_CRITERIA)
print('stereo error:',rmse)

3 posts - 2 participants