Fisheye stereo calibration

Hello there,
i’m desperatly trying to get my fisheye stereo camera working.
So far i’ve tried this code, it doesn’t work with his or mine images.
I’ve tried to rewrite the opencv stereocalibration example code, no luck.
Whatever i do, it always crashes at cv::fisheye::stereoCalibrate.
I generate the objectPoints and imagePoints with the stereoCalibraiton example code, maybe that’s the fault?
I’ve tried different PCs with different openCV configurations (always 4.5.5).
I’ve tried to import stereo-parameters from MATLAB, that crashes as well, no matter if i use cv::fisheye:stereoCalibrate or cv::stereoCalibrate.
i appreciate all the help i can get.

The Code i’m using:

#include <opencv2/core/core.hpp>
#include <opencv2/calib3d/calib3d.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <stdio.h>
#include <filesystem>
#include <string>
#include <vector>
#include <string>
#include <algorithm>
#include <iostream>
#include <iterator>
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>

using namespace std;
using namespace cv;


static int print_help(char** argv)
{
    cout <<
        " Given a list of chessboard images, the number of corners (nx, ny)\n"
        " on the chessboards, and a flag: useCalibrated for \n"
        "   calibrated (0) or\n"
        "   uncalibrated \n"
        "     (1: use stereoCalibrate(), 2: compute fundamental\n"
        "         matrix separately) stereo. \n"
        " Calibrate the cameras and display the\n"
        " rectified results along with the computed disparity images.   \n" << endl;
    cout << "Usage:\n " << argv[0] << " -w=<board_width default=9> -h=<board_height default=6> -s=<square_size default=1.0> <image list XML/YML file default=stereo_calib.xml>\n" << endl;
    return 0;
}




static void StereoCalib(const vector<string>& imagelist, Size boardSize, float squareSize, bool displayCorners = false, bool useCalibrated = true, bool showRectified = true)
{
    if (imagelist.size() % 2 != 0)
    {
        cout << "Error: the image list contains odd (non-even) number of elements\n";
        return;
    }

    const int maxScale = 2;
    // ARRAY AND VECTOR STORAGE:

    vector<vector<Point2f> > imagePoints[2];
    vector<vector<Point3f> > objectPoints;
    Size imageSize;

    int i, j, k, nimages = (int)imagelist.size() / 2;

    imagePoints[0].resize(nimages);
    imagePoints[1].resize(nimages);
    vector<string> goodImageList;

    for (i = j = 0; i < nimages; i++)
    {
        for (k = 0; k < 2; k++)
        {
            const string& filename = imagelist[i * 2 + k];
            Mat img = imread(filename, 0);
            if (img.empty())
                break;
            if (imageSize == Size())
                imageSize = img.size();
            else if (img.size() != imageSize)
            {
                cout << "The image " << filename << " has the size different from the first image size. Skipping the pair\n";
                break;
            }
            bool found = false;
            vector<Point2f>& corners = imagePoints[k][j];
            for (int scale = 1; scale <= maxScale; scale++)
            {
                Mat timg;
                if (scale == 1)
                    timg = img;
                    
                else
                    resize(img, timg, Size(), scale, scale, INTER_LINEAR_EXACT);
                    
                found = findChessboardCorners(timg, boardSize, corners,
                    CALIB_CB_ADAPTIVE_THRESH | CALIB_CB_NORMALIZE_IMAGE);
                if (found)
                {
                    if (scale > 1)
                    {
                        Mat cornersMat(corners);
                        cornersMat *= 1. / scale;
                    }
                    break;
                }
            }
            if (displayCorners)
            {
                cout << filename << endl;
                Mat cimg, cimg1;
                cvtColor(img, cimg, COLOR_GRAY2BGR);
                drawChessboardCorners(cimg, boardSize, corners, found);
                double sf = 640. / MAX(img.rows, img.cols);
                resize(cimg, cimg1, Size(), sf, sf, INTER_LINEAR_EXACT);
                imshow("corners", cimg1);
                char c = (char)waitKey(500);
                if (c == 27 || c == 'q' || c == 'Q') //Allow ESC to quit
                    exit(-1);
            }
            else
                putchar('.');
            if (!found)
                break;
            cornerSubPix(img, corners, Size(11, 11), Size(-1, -1),
                TermCriteria(TermCriteria::COUNT + TermCriteria::EPS,
                    30, 0.01));
        }
        if (k == 2)
        {
            goodImageList.push_back(imagelist[i * 2]);
            goodImageList.push_back(imagelist[i * 2 + 1]);
            j++;
        }
    }
    cout << j << " pairs have been successfully detected.\n";
    nimages = j;
    if (nimages < 2)
    {
        cout << "Error: too little pairs to run the calibration\n";
        return;
    }

    imagePoints[0].resize(nimages);
    imagePoints[1].resize(nimages);
    objectPoints.resize(nimages);

    for (i = 0; i < nimages; i++)
    {
        for (j = 0; j < boardSize.height; j++)
            for (k = 0; k < boardSize.width; k++)
                objectPoints[i].push_back(Point3f(k * squareSize, j * squareSize, 0));
    }

    cout << "Running stereo calibration ...\n";
    Mat testL = imread("D:/Dokumente/FlexyBot/Code/fisheyeStereoCalibration/left2.jpg"),
        testR = imread("D:/Dokumente/FlexyBot/Code/fisheyeStereoCalibration/right2.jpg");

    printf("Starting Calibration\n");
    cv::Matx33d K1, K2, R;
    cv::Vec3d T;
    cv::Vec4d D1, D2;
    int flag = 0;
    flag |= cv::fisheye::CALIB_RECOMPUTE_EXTRINSIC;
    flag |= cv::fisheye::CALIB_CHECK_COND;
    flag |= cv::fisheye::CALIB_FIX_SKEW;
    //flag |= cv::fisheye::CALIB_FIX_K2;
    //flag |= cv::fisheye::CALIB_FIX_K3;
    //flag |= cv::fisheye::CALIB_FIX_K4;

    cv::fisheye::stereoCalibrate(objectPoints, imagePoints[0], imagePoints[1],
        K1, D1, K2, D2, testR.size(), R, T,0,
    cv::TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 100, DBL_EPSILON));

    cv::FileStorage fs1("stereo_params.xml", cv::FileStorage::WRITE);
    fs1 << "K1" << Mat(K1);
    fs1 << "K2" << Mat(K2);
    fs1 << "D1" << D1;
    fs1 << "D2" << D2;
    fs1 << "R" << Mat(R);
    fs1 << "T" << T;
    printf("Done Calibration\n");

    printf("Starting Rectification\n");

    cv::Mat R1, R2, P1, P2, Q;
    cv::fisheye::stereoRectify(K1, D1, K2, D2, testR.size(), R, T, R1, R2, P1, P2,
        Q, CALIB_ZERO_DISPARITY, testR.size(), 0.0, 1.1);

    fs1 << "R1" << R1;
    fs1 << "R2" << R2;
    fs1 << "P1" << P1;
    fs1 << "P2" << P2;
    fs1 << "Q" << Q;

    printf("Done Rectification\n");


 
    Mat rL, rR, tL, tR, kL, dL, kR, dR,pL,pR;

    fisheye::calibrate(objectPoints, imagePoints[0], testL.size(), kL, dL, rL, tL, fisheye::CALIB_ZERO_DISPARITY);
    fisheye::calibrate(objectPoints, imagePoints[1], testR.size(), kR, dR, rR, tR, fisheye::CALIB_ZERO_DISPARITY);
    fisheye::estimateNewCameraMatrixForUndistortRectify(kL, dL, testL.size(), Matx33d::eye(), pL, 1);
    fisheye::estimateNewCameraMatrixForUndistortRectify(kR, dR, testR.size(), Matx33d::eye(), pR, 1);
    cv::fisheye::undistortImage(testL, testL, kL, dL,pL);
    cv::fisheye::undistortImage(testR, testR, kR, dR,pR);

    imshow("Links", testL);
    imshow("Rechts", testR);
    waitKey(0);
    return;

}

static bool readStringList(const string& filename, vector<string>& l)
{
    l.resize(0);
    FileStorage fs(filename, FileStorage::READ);
    if (!fs.isOpened())
        return false;
    FileNode n = fs.getFirstTopLevelNode();
    if (n.type() != FileNode::SEQ)
        return false;
    FileNodeIterator it = n.begin(), it_end = n.end();
    for (; it != it_end; ++it)
        l.push_back((string)*it);
    return true;
}

int main(int argc, char** argv)
{
    Size boardSize;
    string imagelistfn;
    bool showRectified;
    cv::CommandLineParser parser(argc, argv, "{w|9|}{h|6|}{s|1.0|}{nr||}{help||}{@input|stereo_calib.xml|}");
    if (parser.has("help"))
        return print_help(argv);
    showRectified = !parser.has("nr");
    imagelistfn = samples::findFile(parser.get<string>("@input"));
    boardSize.width = parser.get<int>("w");
    boardSize.height = parser.get<int>("h");
    float squareSize = parser.get<float>("s");
    if (!parser.check())
    {
        parser.printErrors();
        return 1;
    }
    vector<string> imagelist;
    bool ok = readStringList(imagelistfn, imagelist);
    if (!ok || imagelist.empty())
    {
        cout << "can not open " << imagelistfn << " or the string list is empty" << endl;
        return print_help(argv);
    }

    StereoCalib(imagelist, boardSize, squareSize, true, true, showRectified);
    return 0;
}