So. Someone helped me … I think she found the solution. I need two flip matrix to handle the two different sizes:
Mat F1 = Mat::eye(3, 3, CV_64FC1);
F1.at<double>(1,1) = -1.0;
F1.at<double>(1,2) = source_ry - 1.0;
Mat F2 = Mat::eye(3, 3, CV_64FC1);
F2.at<double>(1,1) = -1.0;
F2.at<double>(1,2) = target_ry - 1.0;
H = F2.inv() * H * F1;