python - 文本识别与重构 OCR opencv

Question

链接到原始图像
https://ibb.co/0VC6vkX
我目前正在处理一个 OCR 项目。我对图像进行了预处理，然后应用预训练的 EAST 模型进行文本检测。

import cv2
import numpy as np
from imutils.object_detection import non_max_suppression
import matplotlib.pyplot as plt
%matplotlib inline

img=cv2.imread('bw_image.jpg')
model=cv2.dnn.readNet('frozen_east_text_detection.pb')

#Prepare the Image
#use multiple of 32 to set the new image shape
height,width,colorch=img.shape
new_height=(height//32)*32
new_width=(width//32)*32
print(new_height,new_width)

h_ratio=height/new_height
w_ratio=width/new_width
print(h_ratio,w_ratio)

#blob from image helps us to prepare the image
blob=cv2.dnn.blobFromImage(img,1,(new_width,new_height),(123.68,116.78,103.94),True, False)
model.setInput(blob)

#this model outputs geometry and score maps
(geometry,scores)=model.forward(model.getUnconnectedOutLayersNames())

#once we have done geometry and score maps we have to do post processing to obtain the final text boxes
rectangles=[]
confidence_score=[]
for i in range(geometry.shape[2]):
    for j in range(0,geometry.shape[3]):
    
        if scores[0][0][i][j]<0.1:
            continue

        bottom_x=int(j*4 + geometry[0][1][i][j])
        bottom_y=int(i*4 + geometry[0][2][i][j])

        top_x=int(j*4 - geometry[0][3][i][j])
        top_y=int(i*4 - geometry[0][0][i][j])

        rectangles.append((top_x,top_y,bottom_x,bottom_y))
        confidence_score.append(float(scores[0][0][i][j]))

#use nms to get required triangles
final_boxes=non_max_suppression(np.array(rectangles),probs=confidence_score,overlapThresh=0.5)

#finally to display these text boxes let's iterate over them and convert them to the original shape 
#using the ratio we calculated earlier
img_copy=img.copy()

for (x1,y1,x2,y2) in final_boxes:
    
    x1=int(x1*w_ratio)
    y1=int(y1*h_ratio)
    x2=int(x2*w_ratio)
    y2=int(y2*h_ratio)
    
    #to draw the rectangles on the image use cv2.rectangle function
    cv2.rectangle(img_copy,(x1,y1),(x2,y2),(0,255,0),2)

这为我们提供了检测到的文本如下:

现在对于文本识别，我使用了预训练的 opencv CRNN 模型，如下所示:

# Download the CRNN model and Load it
model1 = cv2.dnn.readNet('D:/downloads/crnn.onnx')


# ## Prepare the image
img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

blob = cv2.dnn.blobFromImage(img_gray, scalefactor=1/127.5, size=(100,32), mean=127.5)


# Pass the image to network and extract per-timestep scores
model1.setInput(blob)

scores = model1.forward()
print(scores.shape)

alphabet_set = "0123456789abcdefghijklmnopqrstuvwxyz"
blank = '-'

char_set = blank + alphabet_set


# Decode the scores to text
def most_likely(scores, char_set):
    text = ""
    for i in range(scores.shape[0]):
        c = np.argmax(scores[i][0])
        text += char_set[c]
    return text


def map_rule(text):
    char_list = []
    for i in range(len(text)):
        if i == 0:
            if text[i] != '-':
                char_list.append(text[i])
        else:
            if text[i] != '-' and (not (text[i] == text[i - 1])):
                char_list.append(text[i])
    return ''.join(char_list)


def best_path(scores, char_set):
    text = most_likely(scores, char_set)
    final_text = map_rule(text)
    return final_text


out = best_path(scores, char_set)
print(out)

但是在图像上应用这个模型会得到以下输出:

saetan

我真的不明白。任何人都可以指导文本识别有什么问题。预训练的 CRNN 模型有问题吗？此外，我还想在文本被识别后重新构造文本，它们在原始图像中的构造方式。识别问题解决后，我们有了边界框坐标和识别的文本，那么我们如何准确地重构文本呢？任何帮助将不胜感激。
编辑:我使用了 pytesseract image_to_string() 和 image_to_data() 函数，但它们的性能不佳。如果这个 CRNN 模型不够合适，我是否可以使用任何其他预训练的文本识别模型，以便我可以复制我的 EAST Text Detection 模型的成功。这样我就可以在通过 EAST 模型获得的 coordinates(bounding boxes) 的帮助下准确地重构图像中的文本。

Answer 1

处理庄稼真的很简单，只需稍微改变你的最后一个循环:

import pytesseract
from PIL import Image

...
 
for x1,y1,x2,y2 in final_boxes:
        
    #to draw the rectangles on the image use cv2.rectangle function
    # cv2.rectangle(img_copy,(x1,y1),(x2,y2),(0,255,0),2)
    img_crop = Image.fromarray(img[y1-1: y2+1, x1-1:x2+1])
    text = pytesseract.image_to_string(img_crop, config='--psm 8').strip()
    cv2.putText(img_copy, text, (x1,y1), 0, .7, (0, 0, 255), 2 )