Stereo Rectification shows a black and a gray image?!

Hello everyone,
i want to calibrate my Insta360EVO as a fisheye stereo camera, therefore i calibrated every camera alone for the intrinsic parameters and then i used the stereoCalibration from the fisheye modul, to get the extrinsics. This worked and i have all calibration parameters to undistort my Images. This worked very well, when i used the camera parameters without the extrinsics (rotation and translation). For example i took the parameters from my left camera and undistort these images and saw my image undistort like you can see it in tutorials. Same worked for the right camera. Now when i use “fisheye.stereoRectify” and use the projection to undistored (fisheye.initUndistortRectifyMap) and remap my images, i see only black and gray image. When i print the min and max values from these images i see, there is a span from left:21-22 int and right:182-183 int. This cant be right or? I printed out the projections matrix too, here are the results:

[[1.28744564e+00 0.00000000e+00 6.44267878e+05 0.00000000e+00]
 [0.00000000e+00 1.28744564e+00 6.45199521e+05 0.00000000e+00]
 [0.00000000e+00 0.00000000e+00 1.00000000e+00 0.00000000e+00]]
[[ 1.28744564e+00  0.00000000e+00  6.44267878e+05 -4.23120072e+00]
 [ 0.00000000e+00  1.28744564e+00  6.45199521e+05  0.00000000e+00]
 [ 0.00000000e+00  0.00000000e+00  1.00000000e+00  0.00000000e+00]]

I dont know where is the error. Maybe i undistort or rectify wrong. Can someone help and tell me what can i do to get rectified images, which i can see and are not just black and gray?

i calibrated the camera with 22 Images.

Here is my code test.py:

import  cv2
import numpy as np
import sys
sys.path.append(("../../src/"))
def testUndistort():
    import glob

    K_r = np.load("calibration/left_k.npy")
    D_r = np.load("calibration/left_d.npy")
    frame = cv2.imread(glob.glob("data/calibration_img/*.*")[0],1)
    h, w = frame.shape[:2]
    left, right = frame[:, :w // 2], frame[:, w // 2:]
    height, width = left.shape[:2]
    DIM = (width, height)
    while True:
        map1, map2 = cv2.fisheye.initUndistortRectifyMap(K_r, D_r, np.eye(3), K_r, DIM, cv2.CV_16SC2)
        undistorted_img = cv2.remap(right, map1, map2, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT)

        frame = cv2.resize(undistorted_img, (DIM[0] // 4, DIM[1] // 4), interpolation=cv2.INTER_AREA)
        frame1 = cv2.resize(right, (DIM[0] // 4, DIM[1] // 4), interpolation=cv2.INTER_AREA)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            exit()
        cv2.imshow("undistort", frame)
        cv2.imshow("orig", frame1)

def testRectify():
    import glob

    K_right = np.load("calibration/right_k.npy")
    D_right = np.load("calibration/right_d.npy")
    K_left = np.load("calibration/left_k.npy")
    D_left = np.load("calibration/left_d.npy")
    R = np.load("calibration/stereo_R.npy")
    T = np.load("calibration/stereo_T.npy")

    frame = cv2.imread(glob.glob("data/calibration_img/*.*")[0], 1)
    h, w = frame.shape[:2]
    left, right = frame[:, :w // 2], frame[:, w // 2:]
    left = cv2.cvtColor(left, cv2.COLOR_BGR2GRAY)
    right = cv2.cvtColor(right, cv2.COLOR_BGR2GRAY)
    height, width = left.shape[:2]
    DIM = (width, height)
    R_left, R_right, P1, P2, Q = cv2.fisheye.stereoRectify(
        K1=K_left,
        D1=D_left,
        K2=K_right,
        D2=D_right,
        imageSize=DIM,
        R=R,
        tvec=T,
        flags=cv2.fisheye.CALIB_ZERO_DISPARITY,
        balance=0,
        fov_scale=1.0
    )

    mapx_left, mapy_left = cv2.fisheye.initUndistortRectifyMap(
        K=K_left,
        D=D_left,
        R=R_left,
        P=P1,
        # self.P1,
        size=DIM,
        m1type=cv2.CV_32F)
    # self.mapx_right, self.mapy_right
    mapx_right, mapy_right = cv2.fisheye.initUndistortRectifyMap(
        K=K_right,
        D=D_right,
        R=R_right,
        P=P2,
        # self.P2,
        size=DIM,
        m1type=cv2.CV_32F)


    rectLeft = cv2.remap(left, mapx_left, mapy_left, interpolation=cv2.INTER_LINEAR,
                              borderMode=cv2.BORDER_CONSTANT)
    rectRight = cv2.remap(right, mapx_right, mapy_right, interpolation=cv2.INTER_LINEAR,
                               borderMode=cv2.BORDER_CONSTANT)
    print(P1)
    print(P2)
    print(np.min(mapx_left), np.max(mapx_left))
    print(np.min(mapy_left), np.max(mapy_left))
    print(np.min(mapx_right), np.max(mapx_right))
    print(np.min(mapy_right), np.max(mapy_right))
    print(np.min(rectLeft), np.max(rectLeft))
    print(np.min(rectRight), np.max(rectRight))
    while True:
        frame = cv2.resize(rectLeft, (DIM[0] // 4, DIM[1] // 4), interpolation=cv2.INTER_AREA)
        frame1 = cv2.resize(rectRight, (DIM[0] // 4, DIM[1] // 4), interpolation=cv2.INTER_AREA)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            exit()
        cv2.imshow("rectLeft", frame)
        cv2.imshow("rectRight", frame1)


def testRectifyOther():
    import glob

    K_right = np.load("calibration/right_k.npy")
    D_right = np.load("calibration/right_d.npy")
    K_left = np.load("calibration/left_k.npy")
    D_left = np.load("calibration/left_d.npy")
    R = np.load("calibration/stereo_R.npy")
    T = np.load("calibration/stereo_T.npy")

    frame = cv2.imread(glob.glob("data/calibration_img/*.*")[0], 1)
    h, w = frame.shape[:2]
    left, right = frame[:, :w // 2], frame[:, w // 2:]
    left = cv2.cvtColor(left, cv2.COLOR_BGR2GRAY)
    right = cv2.cvtColor(right, cv2.COLOR_BGR2GRAY)
    height, width = left.shape[:2]
    DIM = (width, height)
    R_left, R_right, P1, P2, Q = cv2.fisheye.stereoRectify(
        K1=K_left,
        D1=D_left,
        K2=K_right,
        D2=D_right,
        imageSize=DIM,
        R=R,
        tvec=T,
        flags=cv2.fisheye.CALIB_ZERO_DISPARITY,
        balance=0,
        fov_scale=1.0
    )

    scaled_K_left = K_left * DIM[0] / 2880
    scaled_K_left[2][2] = 1.0  # Except that K[2][2] is always 1.0

    # use scaled_K, dim2 and balance to determine the final K used to un-distort image
    new_K_left = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify(
        K=scaled_K_left,
        D=D_left,
        image_size=DIM,
        R=R_left,
        balance=0,
        fov_scale=1.0,
    )

    scaled_K_right = K_right * DIM[0] / 2880
    # print(scaled_K)
    scaled_K_right[2][2] = 1.0  # Except that K[2][2] is always 1.0
    new_K_right = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify(
        K=scaled_K_right,
        D=D_right,
        image_size=DIM,
        R=R_right,
        balance=0,
        fov_scale=1.0,
    )


    mapx_left, mapy_left = cv2.fisheye.initUndistortRectifyMap(
        K=K_left,
        D=D_left,
        R=R_left,
        P=new_K_left,
        # self.P1,
        size=DIM,
        m1type=cv2.CV_32F)
    # self.mapx_right, self.mapy_right
    mapx_right, mapy_right = cv2.fisheye.initUndistortRectifyMap(
        K=K_right,
        D=D_right,
        R=R_right,
        P=new_K_right,
        # self.P2,
        size=DIM,
        m1type=cv2.CV_32F)

    rectLeft = cv2.remap(left, mapx_left, mapy_left, interpolation=cv2.INTER_LINEAR,
                              borderMode=cv2.BORDER_CONSTANT)
    rectRight = cv2.remap(right, mapx_right, mapy_right, interpolation=cv2.INTER_LINEAR,
                               borderMode=cv2.BORDER_CONSTANT)
    print(np.min(mapx_left), np.max(mapx_left))
    print(np.min(mapy_left), np.max(mapy_left))
    print(np.min(mapx_right), np.max(mapx_right))
    print(np.min(mapy_right), np.max(mapy_right))
    print(np.min(rectLeft), np.max(rectLeft))
    print(np.min(rectRight), np.max(rectRight))
    while True:
        frame = cv2.resize(rectLeft, (DIM[0] // 4, DIM[1] // 4), interpolation=cv2.INTER_AREA)
        frame1 = cv2.resize(rectRight, (DIM[0] // 4, DIM[1] // 4), interpolation=cv2.INTER_AREA)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            exit()
        cv2.imshow("rectLeft", frame)
        cv2.imshow("rectRight", frame1)



#testUndistort()
testRectify()
#testRectifyOther()