Homography from Camera Poses for Aerial Image Stitching

,
1

Hey,
I am currently working on a real time image stitching visualization of aerial imagery. I got the homogenous transformation of the drone, the related image (which might be a little off due to delay in the logging) and the intrinsic camera matrix. In my opinion that should be possible with corresponding formulas for intrinsic (K) and extrinsic (E_1 and E_2) camera matrices.

intr_inv
intr_inv743×266 6.16 KB

Using the projection formula
x=K * E * P
and some reordering you would get this formula:

x=K * E_1 * E_2 * K_inv * y

But when I used a test code (from the internet) for visualization of changes on different parameters it seems to be independent of the translation (which I also noticed when calculating by hand).
The original test code had a change in the intrinsic camera matrix which made it work as aspected (with relations to translation changes) by switching the last two rows

My question is wether my assumption is wrong and I should maybe use a different method or whether there is some good geometric explanation that I didnt get yet.

The test code:

import cv2
import numpy as np

rotXdeg = 90
rotYdeg = 90
rotZdeg = 90
f = 250
distx = 0
disty = 0
distz = 2
50

def onRotXChange(val):
    global rotXdeg
    rotXdeg = val
def onRotYChange(val):
    global rotYdeg
    rotYdeg = val
def onRotZChange(val):
    global rotZdeg
    rotZdeg = val
def onFchange(val):
    global f
    if val == 0:
        val = 1
    f = val
def onDistChangex(val):
    global distx
    distx=val
def onDistChangey(val):
    global disty
    disty=val
def onDistChangez(val):
    global distz
    distz=val

if __name__ == '__main__':

    #Read input image, and create output image
    src = cv2.imread('./test.png')
    src = cv2.resize(src, (500, 500))
    dst = np.ndarray(shape=src.shape,dtype=src.dtype)

    #Create user interface with trackbars that will allow to modify the parameters of the transformation
    wndname1 = "Source:"
    wndname2 = "WarpPerspective: "
    cv2.namedWindow(wndname1, 1)
    cv2.namedWindow(wndname2, 1)
    cv2.createTrackbar("Rotation X", wndname2, rotXdeg, 180, onRotXChange)
    cv2.createTrackbar("Rotation Y", wndname2, rotYdeg, 180, onRotYChange)
    cv2.createTrackbar("Rotation Z", wndname2, rotZdeg, 180, onRotZChange)
    cv2.createTrackbar("f", wndname2, f, 250, onFchange)
    cv2.createTrackbar("DistanceX", wndname2, distx, 250, onDistChangex)
    cv2.createTrackbar("DistanceY", wndname2, disty, 250, onDistChangey)
    cv2.createTrackbar("DistanceZ", wndname2, distz, 250, onDistChangez)

    #Show original image
    # src_gui = cv2.resize(src, (500, 500))
    # Show original image
    # cv2.imshow(wndname1, src_gui)

    cv2.imshow(wndname1, src)

    h , w = src.shape[:2]

    while True:


        rotX = (rotXdeg - 90)*np.pi/180
        rotY = (rotYdeg - 90)*np.pi/180
        rotZ = (rotZdeg - 90)*np.pi/180

        # Projection 2D -> 3D matrix
        # A1= np.matrix([[1, 0, -w/2],
        #                [0, 1, -h/2],
        #                [0, 0, 0   ],
        #                [0, 0, 1   ]])

        # A1 = np.matrix([[1 / f, 0, -w / (2 * f)],
        #                 [0, 1 / f, -h / (2 * f)],
        #                 [0, 0, 0],
        #                 [0, 0, 1]])
        A1 = np.matrix([[1 / f, 0, -w / (2 * f)],
                        [0, 1 / f, -h / (2 * f)],
                        [0, 0, 1],
                        [0, 0, 0]])

        # Rotation matrices around the X,Y,Z axis
        RX = np.matrix([[1,           0,            0, 0],
                        [0,np.cos(rotX),-np.sin(rotX), 0],
                        [0,np.sin(rotX),np.cos(rotX) , 0],
                        [0,           0,            0, 1]])

        RY = np.matrix([[ np.cos(rotY), 0, np.sin(rotY), 0],
                        [            0, 1,            0, 0],
                        [ -np.sin(rotY), 0, np.cos(rotY), 0],
                        [            0, 0,            0, 1]])

        RZ = np.matrix([[ np.cos(rotZ), -np.sin(rotZ), 0, 0],
                        [ np.sin(rotZ), np.cos(rotZ), 0, 0],
                        [            0,            0, 1, 0],
                        [            0,            0, 0, 1]])

        #Composed rotation matrix with (RX,RY,RZ)
        R = RX * RY * RZ

        # R = np.matrix([[0.9999782461755643, -0.003028341372586611, 0.005846811297004637, 0],
        #                            [0.002907679359659945, 0.9997847227209555, 0.0205428341395248, 0],
        #                            [-0.005907763384087175, -0.02052538902164485, 0.9997717887860413, 0],
        #                            [0, 0, 0, 1]])

        #Translation matrix on the Z axis change dist will change the height
        T = np.matrix([[1,0,0,distx],
                       [0,1,0,disty],
                       [0,0,1,distz],
                       # [0, 0, 1, 0.5],
                       [0,0,0,1]])

        #Camera Intrisecs matrix 3D -> 2D
        A2= np.matrix([[f, 0, w/2,0],
                       [0, f, h/2,0],
                       [0, 0,   1,0]])

        # Final and overall transformation matrix
        H = A2 * (T * (R * A1))
        print("T*R: \n")
        print(T*R)

        # Apply matrix transformation
        cv2.warpPerspective(src, H, (w, h), dst, cv2.INTER_CUBIC)

        # dst_new = cv2.resize(dst, (500, 500))

        # Show the image
        # cv2.imshow(wndname2, dst_new)
        cv2.imshow(wndname2, dst)
        cv2.waitKey(1)