[人工智能]使用opencv和aircv 做图片匹配

需求: 给一张小图查找其在大图中的位置

看效果图:

一本书(正射), 和一本桌上的数(书做了仿射变换了) 使用aircv的模板匹配式无法达到效果, 但是使用特征匹配至少可以找到左上和中心点基本正确, [右下有偏差]

?

另外aircv其中有个函数基本无法通过, 最终将aircv代码中以下修改:

改成:

完整得py脚本如下:

?

# Name: contain_image.py
# Description: small image is in big image 
# Author: CHIJING 20210909
# -*- coding: utf-8 -*-
import numpy as np
import pyautogui
import cv2
import aircv as ac
#import opencv-python
#import opencvpyxl
# 可以通过图片，在屏幕上定位图像所在的位置
# 找到返回的是一个4边距元组 (top, left, width, height)，没有找到返回None
# 全屏幕搜素
def _sift_instance(edge_threshold=100):
    if hasattr(cv2, 'SIFT'):
        return cv2.SIFT_create(edgeThreshold=edge_threshold)
    return cv2.xfeatures2d.SIFT_create(edgeThreshold=edge_threshold)


def sift_count(img):
    sift = _sift_instance()
    kp, des = sift.detectAndCompute(img, None)
    return len(kp)

FLANN_INDEX_KDTREE = 0

def find_all_sift(im_source, im_search, min_match_count=4, maxcnt=0):
    '''
    使用sift算法进行多个相同元素的查找
    Args:
        im_source(string): 图像、素材
        im_search(string): 需要查找的图片
        threshold: 阈值，当相识度小于该阈值的时候，就忽略掉
        maxcnt: 限制匹配的数量

    Returns:
        A tuple of found [(point, rectangle), ...]
        A tuple of found [{"point": point, "rectangle": rectangle, "confidence": 0.76}, ...]
        rectangle is a 4 points list
    '''
    sift = _sift_instance()
    flann = cv2.FlannBasedMatcher({'algorithm': FLANN_INDEX_KDTREE, 'trees': 5}, dict(checks=50))

    kp_sch, des_sch = sift.detectAndCompute(im_search, None)
    if len(kp_sch) < min_match_count:
        return None

    kp_src, des_src = sift.detectAndCompute(im_source, None)
    if len(kp_src) < min_match_count:
        return None

    h, w = im_search.shape[1:]

    result = []
    while True:
        # 匹配两个图片中的特征点，k=2表示每个特征点取2个最匹配的点
        matches = flann.knnMatch(des_sch, des_src, k=2)
        good = []
        for m, n in matches:
            # 剔除掉跟第二匹配太接近的特征点
            if m.distance < 0.9 * n.distance:
                good.append(m)

        if len(good) < min_match_count:
            break

        sch_pts = np.float32([kp_sch[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
        img_pts = np.float32([kp_src[m.trainIdx].pt for m in good]).reshape(-1, 1, 2) 

        # M是转化矩阵
        M, mask = cv2.findHomography(sch_pts, img_pts, cv2.RANSAC, 5.0)
        matches_mask = mask.ravel().tolist()

        # 计算四个角矩阵变换后的坐标，也就是在大图中的坐标
        h, w = im_search.shape[:2]
        pts = np.float32([[0, 0], [0, h-1], [w-1, h-1], [w-1, 0]]).reshape(-1, 1, 2)
        dst = cv2.perspectiveTransform(pts, M)

        # trans numpy arrary to python list
        # [(a, b), (a1, b1), ...]
        pypts = []
        for npt in dst.astype(int).tolist():
            pypts.append(tuple(npt[0]))

        lt, br = pypts[0], pypts[2]
        middle_point = (lt[0] + br[0]) / 2, (lt[1] + br[1]) / 2

        result.append(dict(
            result=middle_point,
            rectangle=pypts,
            confidence=(matches_mask.count(1), len(good)) #min(1.0 * matches_mask.count(1) / 10, 1.0)
        ))

        if maxcnt and len(result) >= maxcnt:
            break
        
        # 从特征点中删掉那些已经匹配过的, 用于寻找多个目标
        qindexes, tindexes = [], []
        for m in good:
            qindexes.append(m.queryIdx) # need to remove from kp_sch
            tindexes.append(m.trainIdx) # need to remove from kp_img

        def filter_index(indexes, arr):
            r = np.ndarray(0, np.float32)
            for i, item in enumerate(arr):
                if i not in qindexes:
                    r = np.append(r, item)
            return r
        kp_src = filter_index(tindexes, kp_src)
        des_src = filter_index(tindexes, des_src)

    return result
pass
#
filename_result1 = 'D:\\pycode\\图像识别\\图片\\A组\\assert_ZhiTuShiTu.png'
filename_screen1 = 'D:\\pycode\\图像识别\\图片\\A组\\2021-09-09_11_29_42assert.jpg'
filename_result1 = 'D:\\pycode\\assert_ZhiTuShiTu.png'
filename_screen1 = 'D:\\pycode\\2021-09-09_11_29_42assert.jpg'
filename_result2 = 'D:\\pycode\\at_inters_Point_Point.png'
filename_screen2 = 'D:\\pycode\\2021-09-09_11_36_08assert.jpg'
filename_result3 = 'D:\\pycode\\20181026153953158.png'
filename_screen3 = 'D:\\pycode\\20181026153930243.png'

20181026153930243
#找小图在大图中的位置, 
def SearchsmallInBigPosition(filename_big, filename_small):
    im_result = cv2.imdecode(np.fromfile(filename_small,dtype=np.uint8),-1)  
    im_screen = cv2.imdecode(np.fromfile(filename_big,dtype=np.uint8),-1)  
    #smallimg = cv2.resize(im_result,(100,100))
    #由于png和jpg格式有差异这里rgb设True会更准确
    pos = ac.find_template(im_screen , im_result  , rgb=True, bgremove=True)
    #p2 =  ac.find_sift( im_result,im_screen )  

    if not pos:
        print('未找到')
        return None, None, None
    top_left = pos['rectangle'][0]  # 左上
    right_bottom = pos['rectangle'][3]  # 右下
    circle_center_pos = pos['result']  # 中心点坐标
    circle_center_pos = tuple(map(int, circle_center_pos))
    #print('准确率:', pos['confidence'])
    #print('中心坐标:',circle_center_pos) 
    #print('左上右下:',top_left, right_bottom)
    return circle_center_pos,top_left,right_bottom
pass

def SearchsmallInBigPositionBySIFT(filename_big, filename_small):
    im_result = cv2.imdecode(np.fromfile(filename_small,dtype=np.uint8),-1)  
    im_screen = cv2.imdecode(np.fromfile(filename_big,dtype=np.uint8),-1)  
    pos = find_all_sift(im_screen,im_result,4,1 )
    #im_result_gray = cv2.cvtColor( im_result,  cv2.COLOR_RGB2GRAY )
    #im_screen_gray = cv2.cvtColor( im_screen,  cv2.COLOR_RGB2GRAY )
    #pos = find_all_sift(im_screen_gray,im_result_gray,4,1 )
    if not pos:
        print('未找到')
        return None, None, None
    top_left = pos[0]['rectangle'][0]  # 左上
    right_bottom = pos[0]['rectangle'][3]  # 右下
    circle_center_pos = pos[0]['result']  # 中心点坐标
    circle_center_pos = tuple(map(int, circle_center_pos))
    #print('准确率:', pos['confidence'])
    #print('中心坐标:',circle_center_pos) 
    #print('左上右下:',top_left, right_bottom)
    return circle_center_pos,top_left,right_bottom


def main():
    
    circle_center_pos,top_left,right_bottom =  SearchsmallInBigPosition(filename_screen1, filename_result1)
    print('模板匹配方法中心坐标:',circle_center_pos) 
    print('模板匹配方法左上右下:',top_left, right_bottom)
    circle_center_pos,top_left,right_bottom =      SearchsmallInBigPositionBySIFT(filename_screen1, filename_result1)
    print('特征点匹配中心坐标:',circle_center_pos) 
    print('特征点匹配左上右下:',top_left, right_bottom)


    circle_center_pos,top_left,right_bottom = SearchsmallInBigPosition(filename_screen2, filename_result2)
    print('模板匹配方法中心坐标:',circle_center_pos) 
    print('模板匹配方法左上右下:',top_left, right_bottom)
    circle_center_pos,top_left,right_bottom = SearchsmallInBigPositionBySIFT(filename_screen2, filename_result2)
    print('特征点匹配中心坐标:',circle_center_pos) 
    print('特征点匹配左上右下:',top_left, right_bottom)

    circle_center_pos,top_left,right_bottom = SearchsmallInBigPosition(filename_screen3, filename_result3)
    print('模板匹配方法中心坐标:',circle_center_pos) 
    print('特征点匹配左上右下:',top_left, right_bottom)

    circle_center_pos,top_left,right_bottom = SearchsmallInBigPositionBySIFT(filename_screen3, filename_result3)
    print('特征点匹配中心坐标:',circle_center_pos) 
    print('特征点匹配左上右下:',top_left, right_bottom)
pass

main()