Compare Detected Object To Template To Determine Angle Of Rotation

1

I am trying to find a way to determine how much rotation a detected object in a scene is compared to the template image. It is not quite right, and need some help with this or if there is a better way, that would be great. Thanks.

I just want to test if the icons are oriented correcly:

example_scene
example_scene520×564 49.8 KB

Here is some code:

import numpy as np
import cv2
from matplotlib import pyplot as plt

def feature_detect_with_orb2(train_img_path, scene_img_path):
  
    # Read the query image as query_img (to find) and train image (to be found in)
    train_img = cv2.imread(train_img_path)
    query_img = cv2.imread(scene_img_path)

    # Convert it to grayscale
    query_img_bw = cv2.cvtColor(query_img, cv2.COLOR_BGR2GRAY)
    train_img_bw = cv2.cvtColor(train_img, cv2.COLOR_BGR2GRAY)

    # Initialize the ORB detector algorithm with increased number of features
    patchSizeEdgeThreshold = 15          
    features = 10000
    scaleFactor = 1.2
    levels = 8
    edgeThreshold = patchSizeEdgeThreshold
    fistLevel = 0
    wta = 2
    scoreType=cv2.ORB_FAST_SCORE
    patchSize = patchSizeEdgeThreshold
    fastThrshold = 15
   
    orb = cv2.ORB_create(
        nfeatures=features,    # Increase the number of features to detect
        scaleFactor=scaleFactor,    # A smaller scale factor to create more levels in the pyramid
        nlevels=levels,          # Increase the number of levels in the pyramid
        edgeThreshold=edgeThreshold,   # Default value, but can be adjusted based on image borders
        firstLevel=fistLevel,       # Default value, the level of the pyramid to start from
        WTA_K=wta,            # The default value, producing binary strings from comparisons of pairs of points
        scoreType=scoreType,  
        patchSize=patchSize,       # Size of the patch used by the oriented BRIEF descriptor
        fastThreshold=fastThrshold    # Threshold for the FAST keypoint detector
    )

    # Now detect the keypoints and compute the descriptors for the query image and train image
    queryKeypoints, queryDescriptors = orb.detectAndCompute(query_img_bw, None)
    trainKeypoints, trainDescriptors = orb.detectAndCompute(train_img_bw, None)

    # Check if descriptors were found
    if queryDescriptors is not None and trainDescriptors is not None:
        # Initialize the Matcher for matching the keypoints and then match the keypoints
       
        if(orb.getWTA_K() > 2):
            matcher = cv2.BFMatcher(cv2.NORM_HAMMING2, crossCheck=True)
        else:
            matcher = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)


        matches = matcher.match(queryDescriptors, trainDescriptors)            

        # Sort the matches based on distance. Less distance means better match
        matches = sorted(matches, key=lambda x: x.distance)

        if len(matches) > 4:
            # Extract location of good matches
            points_query = np.float32([queryKeypoints[m.queryIdx].pt for m in matches]).reshape(-1, 1, 2)
            points_train = np.float32([trainKeypoints[m.trainIdx].pt for m in matches]).reshape(-1, 1, 2)

            # Find homography
            homography, mask = cv2.findHomography(points_query, points_train, cv2.RANSAC)

            if homography is not None:
                # Extract the rotation part of the homography matrix
                rotation_matrix = homography[:3, :3]
                
                # Compute the rotation angle in radians
                theta_radians = np.arctan2(rotation_matrix[1, 0], rotation_matrix[0, 0])
                
                # Convert the rotation angle to degrees
                print('Angle: ' + str(np.degrees(theta_radians)))
   

            # Transform the corners of the query image to the scene image
            h, w = query_img_bw.shape
            corners_query_img = np.float32([[0, 0], [0, h-1], [w-1, h-1], [w-1, 0]]).reshape(-1, 1, 2)
            transformed_corners = cv2.perspectiveTransform(corners_query_img, homography)

            # Draw bounding box in the scene image
            detected_img = cv2.polylines(train_img, [np.int32(transformed_corners)], True, (255,0,0), 3, cv2.LINE_AA)


        # draw the matches to the final image containing both the images
        # the drawMatches() function takes both images and keypoints and outputs the matched query image with its train image
        final_img = cv2.drawMatches(query_img, queryKeypoints, train_img, trainKeypoints, matches[:count], None)

        for i, match in enumerate(matches[:count]):  # Limiting to the top 50 matches for readability
            query_idx = match.queryIdx
            train_idx = match.trainIdx
            query_pt = queryKeypoints[query_idx].pt  # Coordinates in the query (template) image
            train_pt = trainKeypoints[train_idx].pt  # Coordinates in the train (scene) image
            #print(f"Match {i+1}: Query Image Coord = {query_pt}, Train Image Coord = {train_pt}")

        # Resize the final image for display
        final_img = cv2.resize(final_img, (1000,650))

        # Show the final image using matplotlib
        plt.imshow(cv2.cvtColor(final_img, cv2.COLOR_BGR2RGB))
        plt.axis('off')
        plt.show()

         # When done, destroy the windows opened by OpenCV
        cv2.destroyAllWindows()
    else:
        print("Descriptors not found in one or both images.")
2

bad code. lots wrong with it. where did you find that code?

3

Originally generated to get me going. Then modified as I learned some about feature detection.

4

by an AI that was trained on a mountain of bad code on the internet, and made to blurt out the answer instead of thinking about it. that’s how LLMs work these days. they blurt stuff out. they aren’t allowed to revise their responses. so it tends to blurt out stuff it memorized, rather than stuff it spent even a moment thinking about. it’s inclined to say what everyone else says, especially when most of them are wrong. that makes it less likely to scare the villagers but it makes for a terrible research assistant. you have to be smarter than it, you have to know what you’re doing (have the formal education), and you need to always check everything it blurts out.

that code uses a substandard feature descriptor. it also lacks the ratio test. it uses the brute force matcher (flann is perfectly capable).

if you ask your AI to critique that code, you need to start a new context and claim the code was written by a third party. if it knows that it wrote that, it will never admit any mistakes. it’s trained not to. it will also find faults or no faults at all, regardless of truth, depending on any implied opinion you have on whether there are faults or not.

if you do it right, it should be able to find and improve upon at least those three points I mentioned.

1 Like
5

Thanks for the response. I made some changes to a different feature descriptor. I only tested it on one sample, but if did find the object, and the angle was within the expected range (near zero).

def feature_detection_with_angle(train_img_path, scene_img_path, rationThreshold = 0.75):
    sceneImage = cv2.imread(scene_img_path)
    templateImage = cv2.imread(train_img_path)

    sceneImage = cv2.cvtColor(sceneImage, cv2.COLOR_BGR2GRAY)
    templateImage = cv2.cvtColor(templateImage, cv2.COLOR_BGR2GRAY)

    sift = cv2.SIFT_create()
 
    kpscene, descr1 = sift.detectAndCompute(sceneImage,None)
    kptemplate, descr2 = sift.detectAndCompute(templateImage,None)
 
    FLANN_INDEX_KDTREE = 0
    index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5)
    search_params = dict(checks=50) 

    matcher = cv2.FlannBasedMatcher(index_params, search_params)
    matches = matcher.knnMatch(descr1,descr2,k=2)

    pointsTemplate = []
    pointsScene = []
    goodMatches = []      
    for i,(m,n) in enumerate(matches):
        if( m.distance < rationThreshold * n.distance):
            goodMatches.append(m)
            pointsScene = kpscene[m.queryIdx].pt
            pointsTemplate = kptemplate[m.trainIdx].pt

    if len(goodMatches) > 4:
        pointsSceneGood = np.float32([kpscene[m.queryIdx].pt for m in goodMatches])[:10]
        pointsTemplateGood = np.float32([kptemplate[m.trainIdx].pt for m in goodMatches])[:10]        

        homography, mask = cv2.findHomography(pointsTemplateGood, pointsSceneGood, cv2.RANSAC)

        if homography is not None:
            theta_radians = np.arctan2(homography[1, 0], homography[0, 0])                
            print('Angle: ' + str(np.degrees(theta_radians)))   

            h, w = sceneImage.shape
            corners_query_img = np.float32([[0, 0], [0, h-1], [w-1, h-1], [w-1, 0]]).reshape(-1, 1, 2)
            transformed_corners = cv2.perspectiveTransform(corners_query_img, homography)            
            
    img3 = cv2.drawMatches(sceneImage,kpscene,templateImage,kptemplate,goodMatches,None,flags=cv2.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS)
    plt.imshow(img3),plt.show()