python - 如何转换图像以使投影图像与原始图像相同

Question

问题陈述:图像 A 通过投影仪投影，经过显微镜，投影图像通过与图像 B 相同的显微镜通过相机捕获。由于光学元件，B 相对于A. 现在，我需要在投影之前将 A 转换为 A'，以使 B 尽可能接近 A。

初始方法:我采用棋盘图案并以不同角度(36、72、108、... 324 度)旋转它并投影以获得一系列 A 图像和 B 图像。我使用 OpenCV 的 CalibrateCamera2、InitUndistortMap 和 Remap 函数将 B 转换为 B'。但是 B' 与 A 相去甚远，并且与 B 相当相似(尤其是存在大量的旋转和剪切没有得到纠正)。

代码(在 Python 中)如下。我不确定我是否在做一些愚蠢的事情。有关正确方法的任何想法？

import pylab
import os
import cv
import cv2
import numpy

# angles - the angles at which the picture was rotated 
angles = [0, 36, 72, 108, 144, 180, 216, 252, 288, 324]
# orig_files - list of original picture files used for projection
orig_files =  ['../calibration/checkerboard/orig_%d.png' % (angle) for angle in angles]
# img_files - projected image captured by camera
img_files = ['../calibration/checkerboard/imag_%d.bmp' % (angle) for angle in angles]
# Load the images 
images = [cv.LoadImage(filename) for filename in img_files]
orig_images = [cv.LoadImage(filename) for filename in orig_files]

# Convert to grayscale
gray_images = [cv.CreateImage((src.height, src.width), cv.IPL_DEPTH_8U, 1) for src in images]
for ii in range(len(images)):
    cv.CvtColor(images[ii], gray_images[ii], cv.CV_RGB2GRAY)
gray_orig = [cv.CreateImage((src.height, src.width), cv.IPL_DEPTH_8U, 1) for src in orig_images]
for ii in range(len(orig_images)):
    cv.CvtColor(orig_images[ii], gray_orig[ii], cv.CV_RGB2GRAY)

# The number of ranks and files in the chessboard. OpenCV considers
# the height and width of the chessboard to be one less than these,
# respectively.
rank_count = 11
file_count = 10

# Try to detect the corners of the chessboard. For each image,
# FindChessboardCorners returns (found, corner_points). found is True
# even if it managed to detect only a subset of the actual corners.
img_corners = [cv.FindChessboardCorners(img, (rank_count-1, file_count-1)) for img in gray_images]
orig_corners = [cv.FindChessboardCorners(img, (rank_count-1,file_count-1)) for img in gray_orig]

# The total number of corners will be (rank_count-1)x(file_count-1),
# but if some parts of the image are too blurred/distorted,
# FindChessboardCorners detects only a subset of the corners. In that
# case, DrawChessboardCorners will raise a TypeError.
orig_corner_success = []
ii = 0
for (found, corners) in orig_corners:
    if found and (len(corners) == (rank_count - 1) * (file_count - 1)):
        orig_corner_success.append(ii)
    else:
        print orig_files[ii], ': could not find correct corners: ', len(corners)
    ii += 1
ii = 0
img_corner_success = []
for (found, corners) in img_corners:
    if found and (len(corners) == (rank_count-1) * (file_count-1)) and (ii in orig_corner_success):
        img_corner_success.append(ii)
    else:
        print img_files[ii], ': Number of corners detected is wrong:', len(corners)
    ii += 1

# Here we compile all the corner coordinates into single arrays    
image_points = []
obj_points = []
for ii in img_corner_success:
    obj_points.extend(orig_corners[ii][1])
    image_points.extend(img_corners[ii][2])        
image_points = cv.fromarray(numpy.array(image_points, dtype='float32'))
obj_points = numpy.hstack((numpy.array(obj_points, dtype='float32'), numpy.zeros((len(obj_points), 1), dtype='float32')))
obj_points = cv.fromarray(numpy.array(obj_points, order='C'))

point_counts = numpy.ones((len(img_corner_success), 1), dtype='int32') * ((rank_count-1) * (file_count-1))
point_counts = cv.fromarray(point_counts)
# Create the output parameters
cam_mat = cv.CreateMat(3, 3, cv.CV_32FC1)
cv.Set2D(cam_mat, 0, 0, 1.0)
cv.Set2D(cam_mat, 1, 1, 1.0)
dist_mat = cv.CreateMat(5, 1, cv.CV_32FC1)
rot_vecs = cv.CreateMat(len(img_corner_success), 3, cv.CV_32FC1)
tran_vecs = cv.CreateMat(len(img_corner_success), 3, cv.CV_32FC1)
# Do the camera calibration
x = cv.CalibrateCamera2(obj_points, image_points, point_counts, cv.GetSize(gray_images[0]), cam_mat, dist_mat, rot_vecs, tran_vecs)
# Create the undistortion map
xmap = cv.CreateImage(cv.GetSize(images[0]), cv.IPL_DEPTH_32F, 1)
ymap = cv.CreateImage(cv.GetSize(images[0]), cv.IPL_DEPTH_32F, 1)
cv.InitUndistortMap(cam_mat, dist_mat, xmap, ymap)
# Now undistort all the images and same them
ii = 0
for tmp in images:
    print img_files[ii]
    image = cv.GetImage(tmp)
    t = cv.CloneImage(image)
    cv.Remap(t, image, xmap, ymap, cv.CV_INTER_LINEAR + cv.CV_WARP_FILL_OUTLIERS, cv.ScalarAll(0))
    corrected_file = os.path.join(os.path.dirname(img_files[ii]), 'corrected_%s' % (os.path.basename(img_files[ii])))
    cv.SaveImage(corrected_file, image)
    print 'Saved corrected image to', corrected_file
    ii += 1

这是图像 - A、B 和 B' 实际上我不认为 Remap 真的在做任何事情!

Answer 1

我终于解决了。有几个问题:

原始图像的大小不同。捕获的图像也不是。因此，来自一对的仿射变换不适用于另一对。我将它们全部调整为相同的大小。

相机校准后的 Undistort 不足以用于旋转和剪切。适当的做法是仿射变换。并且最好取棋盘的三个角作为计算变换矩阵的点(相对误差较小)。

这是我的工作代码(我正在转换原始图像并保存它们以显示实际计算的转换矩阵将原始图像映射到捕获的图像):

import pylab
import os
import cv
import cv2
import numpy

global_object_points = None
global_image_points = None
global_captured_corners = None
global_original_corners = None
global_success_index = None

global_font = cv.InitFont(cv.CV_FONT_HERSHEY_PLAIN, 1.0, 1.0)

def get_camera_calibration_data(original_image_list, captured_image_list, board_width, board_height):
    """Get the map for undistorting projected images by using a list of original chessboard images and the list of images that were captured by camera.

    original_image_list - list containing the original images (loaded as OpenCV image).

    captured_image_list - list containing the captured images.

    board_width - width of the chessboard (number of files - 1)

    board_height - height of the chessboard (number of ranks - 1)

    """
    global global_object_points
    global global_image_points
    global global_captured_corners
    global global_original_corners
    global global_success_index
    print 'get_undistort_map'
    corner_count = board_width * board_height
    # Try to detect the corners of the chessboard. For each image,
    # FindChessboardCorners returns (found, corner_points). found is
    # True even if it managed to detect only a subset of the actual
    # corners.  NOTE: according to
    # http://opencv.willowgarage.com/wiki/documentation/cpp/calib3d/findChessboardCorners,
    # no need for FindCornerSubPix after FindChessBoardCorners
    captured_corners = [cv.FindChessboardCorners(img, (board_width, board_height)) for img in captured_image_list]
    original_corners = [cv.FindChessboardCorners(img, (board_width, board_height)) for img in original_image_list]
    success_captured = [index for index in range(len(captured_image_list))
                        if captured_corners[index][0] and len(captured_corners[index][1]) == corner_count]
    success_original = [index for index in range(len(original_image_list))
                        if original_corners[index][0] and len(original_corners[index][2]) == corner_count]
    success_index = [index for index in success_captured if (len(captured_corners[index][3]) == corner_count) and (index in success_original)]
    global_success_index = success_index
    print global_success_index
    print 'Successfully found corners in image #s.', success_index
    cv.NamedWindow('Image', cv.CV_WINDOW_AUTOSIZE)
    for index in success_index:
        copy = cv.CloneImage(original_image_list[index])
        cv.DrawChessboardCorners(copy, (board_width, board_height), original_corners[index][4], corner_count)
        cv.ShowImage('Image', copy)
        a = cv.WaitKey(0)
        copy = cv.CloneImage(captured_image_list[index])
        cv.DrawChessboardCorners(copy, (board_width, board_height), captured_corners[index][5], corner_count)
        cv.ShowImage('Image', copy)
        a = cv.WaitKey(0)
    cv.DestroyWindow('Image')
    if not success_index:
        return
    global_captured_corners = [captured_corners[index][6] for index in success_index]
    global_original_corners = [original_corners[index][7] for index in success_index]
    object_points = cv.CreateMat(len(success_index) * (corner_count), 3, cv.CV_32FC1)
    image_points = cv.CreateMat(len(success_index) * (corner_count), 2, cv.CV_32FC1)
    global_object_points = object_points
    global_image_points = image_points
    point_counts = cv.CreateMat(len(success_index), 1, cv.CV_32SC1)
    for ii in range(len(success_index)):        
        for jj in range(corner_count):
            cv.Set2D(object_points, ii * corner_count + jj, 0, float(jj/board_width))
            cv.Set2D(object_points, ii * corner_count + jj, 1, float(jj%board_width))
            cv.Set2D(object_points, ii * corner_count + jj, 2, float(0.0))
            cv.Set2D(image_points, ii * corner_count + jj, 0, captured_corners[success_index[ii]][8][jj][0])
            cv.Set2D(image_points, ii * corner_count + jj, 1, captured_corners[success_index[ii]][9][jj][10])
        cv.Set1D(point_counts, ii, corner_count)
    # Create the output parameters    
    camera_intrinsic_mat = cv.CreateMat(3, 3, cv.CV_32FC1)
    cv.Set2D(camera_intrinsic_mat, 0, 0, 1.0)
    cv.Set2D(camera_intrinsic_mat, 1, 1, 1.0)
    distortion_mat = cv.CreateMat(5, 1, cv.CV_32FC1)
    rotation_vecs = cv.CreateMat(len(success_index), 3, cv.CV_32FC1)
    translation_vecs = cv.CreateMat(len(success_index), 3, cv.CV_32FC1)
    print 'Before camera clibration'
    # Do the camera calibration
    cv.CalibrateCamera2(object_points, image_points, point_counts, cv.GetSize(original_image_list[0]), camera_intrinsic_mat, distortion_mat, rotation_vecs, translation_vecs)
    return (camera_intrinsic_mat, distortion_mat, rotation_vecs, translation_vecs)

if __name__ == '__main__':
    # angles - the angles at which the picture was rotated 
    angles = [0, 36, 72, 108, 144, 180, 216, 252, 288, 324]
    # orig_files - list of original picture files used for projection
    orig_files =  ['../calibration/checkerboard/o_orig_%d.png' % (angle) for angle in angles]
    # img_files - projected image captured by camera
    img_files = ['../calibration/checkerboard/captured_imag_%d.bmp' % (angle) for angle in angles]

    # orig_files = ['o%d.png' % (angle) for angle in range(10, 40, 10)]
    # img_files = ['d%d.png' % (angle) for angle in range(10, 40, 10)]
    # Load the images
    print 'Loading images'
    captured_images = [cv.LoadImage(filename) for filename in img_files]
    orig_images = [cv.LoadImage(filename) for filename in orig_files]
    # Convert to grayscale
    gray_images = [cv.CreateImage((src.height, src.width), cv.IPL_DEPTH_8U, 1) for src in captured_images]
    for ii in range(len(captured_images)):
        cv.CvtColor(captured_images[ii], gray_images[ii], cv.CV_RGB2GRAY)
        cv.ShowImage('win', gray_images[ii])
        cv.WaitKey(0)
    cv.DestroyWindow('win')
    gray_orig = [cv.CreateImage((src.height, src.width), cv.IPL_DEPTH_8U, 1) for src in orig_images]
    for ii in range(len(orig_images)):
        cv.CvtColor(orig_images[ii], gray_orig[ii], cv.CV_RGB2GRAY)

    # The number of ranks and files in the chessboard. OpenCV considers
    # the height and width of the chessboard to be one less than these,
    # respectively.
    rank_count = 10
    file_count = 11
    camera_intrinsic_mat, distortion_mat, rotation_vecs, translation_vecs, = get_camera_calibration_data(gray_orig, gray_images, file_count-1, rank_count-1)
    xmap = cv.CreateImage(cv.GetSize(captured_images[0]), cv.IPL_DEPTH_32F, 1)
    ymap = cv.CreateImage(cv.GetSize(captured_images[0]), cv.IPL_DEPTH_32F, 1)
    cv.InitUndistortMap(camera_intrinsic_mat, distortion_mat, xmap, ymap)
    # homography = cv.CreateMat(3, 3, cv.CV_32F)
    map_matrix = cv.CreateMat(2, 3, cv.CV_32F)
    source_points = (global_original_corners[0][0], global_original_corners[0][file_count-2], global_original_corners[0][(rank_count-1) * (file_count-1) -1])
    image_points = (global_captured_corners[0][0], global_captured_corners[0][file_count-2], global_captured_corners[0][(rank_count-1) * (file_count-1) -1])
    # cv.GetPerspectiveTransform(source, target, homography)
    cv.GetAffineTransform(source_points, image_points, map_matrix)
    ii = 0
    cv.NamedWindow('OriginaImage', cv.CV_WINDOW_AUTOSIZE)
    cv.NamedWindow('CapturedImage', cv.CV_WINDOW_AUTOSIZE)
    cv.NamedWindow('FixedImage', cv.CV_WINDOW_AUTOSIZE)
    for image in gray_images:
        # The affine transform should be ideally calculated once
        # outside this loop, but as the transform looks different for
        # each image, I'll just calculate it independently to see the
        # applicability
        try:
            # Try to find ii in the list of successful corner
            # detection indices and if found, use the corners for
            # computing the affine transformation matrix. This is only
            # required when the optics changes between two
            # projections, which should not happend.
            jj = global_success_index.index(ii)
            source_points = [global_original_corners[jj][0], global_original_corners[jj][rank_count-1], global_original_corners[jj][-1]]
            image_points = [global_captured_corners[jj][0], global_captured_corners[jj][rank_count-1], global_captured_corners[jj][-1]]
            cv.GetAffineTransform(source_points, image_points, map_matrix)
            print '---------------------------------------------------------------------'
            print orig_files[ii], '<-->', img_files[ii]
            print '---------------------------------------------------------------------'
            for kk in range(len(source_points)):
                print source_points[kk]
                print image_points[kk]
        except ValueError:
            # otherwise use the last used transformation matrix
            pass

        orig = cv.CloneImage(orig_images[ii])        
        cv.PutText(orig, '%s: original' % (os.path.basename(orig_files[ii])), (100, 100), global_font, 0.0)
        cv.ShowImage('OriginalImage', orig)
        target = cv.CloneImage(image)
        target.origin = image.origin
        cv.SetZero(target)
        cv.Remap(image, target, xmap, ymap, cv.CV_INTER_LINEAR + cv.CV_WARP_FILL_OUTLIERS, cv.ScalarAll(0))
        cv.PutText(target, '%s: remapped' % (os.path.basename(img_files[ii])), (100, 100), global_font, 0.0)
        cv.ShowImage('CapturedImage', target)
        target = cv.CloneImage(orig_images[ii])
        cv.SetZero(target)
        cv.WarpAffine(orig_images[ii], target, map_matrix, cv.CV_INTER_LINEAR | cv.CV_WARP_FILL_OUTLIERS)
        corrected_file = os.path.join(os.path.dirname(img_files[ii]), 'corrected_%s' % (os.path.basename(img_files[ii])))
        cv.SaveImage(corrected_file, target)
        print 'Saved corrected image to', corrected_file
        # cv.WarpPerspective(image, target, homography, cv.CV_INTER_LINEAR | cv.CV_WARP_INVERSE_MAP | cv.CV_WARP_FILL_OUTLIERS)        
        cv.PutText(target, '%s: perspective-transformed' % (os.path.basename(img_files[ii])), (100, 100), global_font, 0.0)
        cv.ShowImage('FixedImage', target)
        print '==================================================================='
        cv.WaitKey(0)
        ii += 1
    cv.DestroyWindow('OriginalImage')
    cv.DestroyWindow('CapturedImage')
    cv.DestroyWindow('FixedImage')

和图像:

原来的:


捕获的图像:


仿射变换的原始图像:


现在应用于原始图像的逆变换应该可以解决问题。