Why I don't get good output for stereo rectification?

[A.I.E]

I want to do some stereo rectify experiments, but I do not get a reasonable output for the next two images.

the output result:

my code:

#include <iostream>
#include <stdint.h>
#include <iostream>
#include <memory>
#include <chrono>
#include <thread>
#include <fstream>
#include <random>
#include <unistd.h>
#include <math.h>


#include <Eigen/Core>
#include <Eigen/Geometry>
#include <Eigen/Dense>



#include <opencv2/opencv.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/calib3d.hpp>
#include <opencv2/core.hpp>
#include <opencv2/sfm.hpp>
#include <opencv2/sfm/robust.hpp>
#include <opencv2/core/eigen.hpp>
#include <opencv2/core/quaternion.hpp>


#include <librealsense2/rs.hpp>


#include <pangolin/pangolin.h>

#include <sophus/se3.hpp>


#include "camera/AbstractCamera.hpp"
#include "camera/RS_Camera.hpp"
#include "camera/LocalVideo.hpp"
#include "camera/USB_Camera.hpp"
#include "camera/ImageSet.h"
#include "utils/FrameMetadata.h"
#include "utils/utils.h"
#include "evaluation/evaluation_method.h"








int main(int argc, char** argv)
{
    // prepare
    std::string imgpath1 = "/home/vipl/tmp/slam_test_imgs/epfl-fountain/0003.png";
    std::string imgpath2 = "/home/vipl/tmp/slam_test_imgs/epfl-fountain/0007.png";

    cv::Mat img2 = cv::imread(imgpath1);
    cv::Mat img1 = cv::imread(imgpath2);



    /*
    // 0000.png
    cv::Mat rotation1 = (cv::Mat_<double>(3,3)<<0.450927,-0.0945642,-0.887537,
            -0.892535,-0.0401974,-0.449183,
            0.00679989,0.994707,-0.102528);
    */


    /*
    // 0002.png
    cv::Mat rotation2 = (cv::Mat_<double>(3,3)<<0.666779,-0.0831384,-0.740603,
            -0.74495,-0.0459057,-0.665539,
            0.021334,0.99548,-0.0925429);
    */

    // 0003.png
    cv::Mat rotation2 = (cv::Mat_<double>(3,3)<<
            0.795163,-0.050195,-0.604314,
            -0.606377,-0.0736593,-0.791759,
            -0.00477103,0.996019,-0.0890082);
    cv::Mat translation2 = (cv::Mat_<double>(3,1)<<-10.8142,-4.53704,0.122293);

    /*
    // 0004.png
    cv::Mat rotation2 = (cv::Mat_<double>(3,3)<<0.890856,-0.0211638,-0.453793,
            -0.454283,-0.0449857,-0.889721,
            -0.00158434,0.998763,-0.0496901);
    */

    // 0007.png
    cv::Mat rotation1 = (cv::Mat_<double>(3,3)<<
        0.995535,0.003721,0.0943235,
        0.0943815,-0.02118,-0.995311,
        -0.00170578,0.999769,-0.0214366);
    cv::Mat translation1 = (cv::Mat_<double>(3,1)<<-17.6302,-3.36186,0.0325247);

    cv::Mat cam_intrinsics;
    camIntrinsicsRead(cam_intrinsics,"epfl");

    cv::Mat rotation = rotation2*rotation1.inv();
    cv::Mat translation = translation2-translation1;

    cv::Vec3d baseline_c1c2 = translation;


    cv::Mat p1,p2,res;
    cv::stereoRectify(
            cam_intrinsics,cv::Matx<double, 1,5>(0,0,0,0),
            cam_intrinsics,cv::Matx<double, 1,5>(0,0,0,0),
            img1.size(),
            rotation,translation,
            rotation1,rotation2,
            p1,p2,res);
    cv::Mat lmap1 = cv::Mat::zeros(img1.rows,img1.cols, CV_32FC2);
    //cv::Mat lmap2 = cv::Mat::zeros(img1.rows*img1.cols, 2, CV_64FC1);
    cv::Mat lmap2;
    cv::Mat rmap1 = cv::Mat::zeros(img1.rows,img1.cols, CV_32FC2);
    // cv::Mat rmap2 = cv::Mat::zeros(img1.rows*img1.cols, 2, CV_64FC1);
    cv::Mat rmap2;



    cv::Mat lhomography = cam_intrinsics*rotation1.t()*cam_intrinsics.inv();
    cv::Mat rhomography = cam_intrinsics*rotation2.t()*cam_intrinsics.inv();
    cv::Mat lres, rres;
    // cv::Mat lres = cv::Mat::zeros(img1.rows,img1.cols,CV_8UC3);
    // cv::Mat rres = cv::Mat::zeros(img2.rows,img2.cols,CV_8UC3);
    for(int i=0;i<img1.rows;i++){
        for(int j=0;j<img1.cols;j++){

            cv::Vec3d lt = cv::Mat(lhomography*cv::Vec3d(j,i,1));   
            // lt = lt/lt[2];
            cv::Vec3d rt = cv::Mat(rhomography*cv::Vec3d(j,i,1));
            // rt = rt/rt[2];

            if(lt[0]<0 || lt[0]>img1.cols || lt[1]<0 || lt[1]>img1.rows){
                // std::cout<<lmap1.at<cv::Vec2d>(j,i)<<std::endl;
                lmap1.at<cv::Vec2f>(i,j)[0] = 0;
                lmap1.at<cv::Vec2f>(i,j)[1] = 0;
            }
            else{
                lmap1.at<cv::Vec2f>(i,j)[0] = lt[0];
                lmap1.at<cv::Vec2f>(i,j)[1] = lt[1];
            }

            if(rt[0]<0 || rt[0]>img1.cols || rt[1]<0 || rt[1]>img1.rows){
                rmap1.at<cv::Vec2f>(i,j)[0] = 0;
                rmap1.at<cv::Vec2f>(i,j)[1] = 0;
            }
            else{
                rmap1.at<cv::Vec2f>(i,j)[0] = rt[0];
                rmap1.at<cv::Vec2f>(i,j)[1] = rt[1];
            }

            /*
            cv::Scalar rtn;
            floatIndex(img1, lt[0], lt[1], rtn);
            lres.at<cv::Scalar_<uchar>>(j,i) = rtn;

            floatIndex(img2, rt[0], rt[1], rtn);
            rres.at<cv::Scalar_<uchar>>(j,i) = rtn;
             */
        }
    }

    cv::remap(img1, lres, lmap1, lmap2, cv::INTER_LINEAR);
    cv::remap(img2, rres, rmap1, rmap2, cv::INTER_LINEAR);

    cv::Mat concatres;
    cv::hconcat(lres,rres,concatres);
    cv::resize(concatres,concatres,cv::Size(1600,900));

    cv::imshow("test",concatres);
    cv::waitKey(0);

    // 

    cv::imwrite("/home/vipl/tmp/reslimg.png",lres);
    cv::imwrite("/home/vipl/tmp/resrimg.png",rres);
    cv::imwrite("/home/vipl/tmp/resconcat.png",concatres);

    return 0;
}

[A.I.E]

this is the origin image, comes from EPFL-fountain:

[A.I.E]