基于FPGA的车牌识别系统

程序基于Xilinx公司的Pynq-Z2开发板,使用opencv库完成车牌识别.

项目背景和设计目的

• 车牌识别系统是计算机视频图像识别技术在车辆牌照识别中的一种应用，在高速公路、停车场、小区、道路等环境下有着广泛的应用。

• 我们希望能够充分利用PYNQ的内部资源，运用Python语言的程序设计和OpenCV计算机视觉库，设计出一个较为可靠的车牌识别系统，将输出结果显示到显示器上，包含车牌号码和车速等信息。

• 对于停车场门口或收费站等应用场景,本系统还可以直接控制舵机,用于限制车辆的进出

产品特点

• 运用了Python语言的程序设计和OpenCV计算机视觉库，使用了ZYNQ-7020芯片，将ARM处理器嵌入到FPGA中，这样将ARM处理器的优势和FPGA的优势结合到了一起。

• 研究基于FPGA+ARM平台的车牌识别系统可利用ARM处理器在操作系统方面的优势，能够实现良好的人机交互功能；又利用FPGA在并行算法加速、可动态重配置的特点，实现硬件加速、增加系统的灵活性，提高车牌识别速度。

• 使用USB外设单目摄像头读取图像信息，能准确、清晰地搜集到车辆图像，图像分辨率达到设计要求。使用外设显示屏输出显示车牌信息。

• 外接舵机，模拟小区出入口和高速公路收费站。

硬件平台

使用Xilinx公司的PYNQ-Z2开发板，搭载ZYNQ-7020芯片和ARM-A9双核处理，使用Python语言和Verilog HDL硬件描述语言对板子进行描述。

性能和成本

现场可编程门阵列（FPGA）和图形处理器（GPU）的性能对比。FPGA的功耗远小于GPU，并且FPGA可以根据特定的应用去编程硬件，虽然FPGA的运算速度相较GPU较慢，总体上，谁在机器视觉方面具备优势是一个需要深入研究的课题，众多企业对FPGA的应用前景比较看好，具备较强的研究价值和应用前景。

FPGA的造价相对GPU较低，在未来，随着技术的成熟和广泛的应用，FPGA的制造成本和销售价格推测均会进一步降低，在现在和将来能够带来的经济收益会高于GPU，在这方面，FPGA应用于机器视觉领域具备很强的潜在的社会经济效益。

整体框图

FPGA处理模块

算法流程图

运行结果

改进

图像处理的拓展。在本次设计中，尚未加入车型识别/驾驶员特征（如有无系安全带等）等内容的读取，在进一步研究中可以考虑增加上述内容的研究和识别。

加速模块尚未完善,加速效果不显著。

物联网方向的深入。在本次设计中，尚未设置联网比对功能，如果有这部分的设计，那么我们可以实时与公安的数据进行比对，如车辆颜色等特征判断是否为套牌，车牌是否系伪造（数据库中无该车牌），是否是被盗车辆、未缴纳交通强制保险车辆、肇事逃逸车辆等，功能会更加完善、齐全。

Python代码

以下为python代码,但是并不包含KNN模型的数据,完整的工程请看:

https://github.com/chenjianqu/Pynq-LicensePlateRecognition

from pynq.overlays.base import BaseOverlay
from pynq.lib.video import *
from matplotlib import pyplot as plt
import numpy as np
import cv2
import time
import math
from time import sleep
from pynq.lib.pmod import Pmod_PWM
from pynq.lib.pmod import PMODAbase = BaseOverlay("base.bit")
pwm = Pmod_PWM(PMODA, 0)#读取字典和数据
with open('knn_dict_letter.txt','r') as f:labelDict = eval(f.read())
with np.load('knn_data.npz') as data:train_data=data['train_data']train_labels=data['train_labels']
with np.load('knn_data_zh.npz') as data:train_data_zh=data['train_data_zh']train_labels_zh=data['train_labels_zh']# 定义分类器
cascade_path = 'cascade.xml'
cascade_car_path = 'cars.xml'
car_cascade = cv2.CascadeClassifier(cascade_car_path)
cascade = cv2.CascadeClassifier(cascade_path)# monitor configuration: 640*480 @ 24Hz
Mode = VideoMode(640,480,24)
hdmi_out = base.video.hdmi_out
hdmi_out.configure(Mode,PIXEL_BGR)
hdmi_out.start()# monitor (output) frame buffer size
frame_out_w = 640
frame_out_h = 480scale=400/480 #比例系数PLATE_WIDTH=0.44Hmin=100
Hmax=124
Smin=120
Smax=255
Vmin=120
Vmax=255reg=["ShangHaiF19911","ShangHaiB6N866","NingXia1B6N86","ShangHaiB6N866","ShangHaiB8N866","ShangHaiB0N866","NingXiaB19873","HeNanP8A629"]isRecognize=3print('Read Data Done')def KeyCallBack():if(base.buttons[0].read()==1):isRecognize=0elif(base.buttons[1].read()==1):isRecognize=1elif(base.buttons[2].read()==1):isRecognize=2elif(base.buttons[3].read()==1):isRecognize=3#精确的定位车牌
def locateAccuracy(img):frame=cv2.cvtColor(img,cv2.COLOR_BGR2HSV)height = frame.shape[0]width = frame.shape[1]top=0button=height-1left=0right=width-1row=0while(row<height):col=0count=0while(col<width):if((frame[row,col,0]>=Hmin and frame[row,col,0]<=Hmax) and (frame[row,col,1]>=Smin and frame[row,col,1]<=Smax) and (frame[row,col,2]>=Vmin and frame[row,col,2]<=Vmax)):count+=1col+=1if(count/width>0.6):top=rowbreakrow+=1row=buttonwhile(row>0):col=0count=0while(col<width):if((frame[row,col,0]>=Hmin and frame[row,col,0]<=Hmax) and (frame[row,col,1]>=Smin and frame[row,col,1]<=Smax) and (frame[row,col,2]>=Vmin and frame[row,col,2]<=Vmax)):count+=1col+=1if(count/width>0.6):button=rowbreakrow-=1col=rightwhile(col>0):row=0count=0while(row<height):if((frame[row,col,0]>=Hmin and frame[row,col,0]<=Hmax) and (frame[row,col,1]>=Smin and frame[row,col,1]<=Smax) and (frame[row,col,2]>=Vmin and frame[row,col,2]<=Vmax)):count+=1row+=1if(count/height>0.6):right=colbreakcol-=1col=leftwhile(col<width):row=0count=0while(row<height):if((frame[row,col,0]>=Hmin and frame[row,col,0]<=Hmax) and (frame[row,col,1]>=Smin and frame[row,col,1]<=Smax) and (frame[row,col,2]>=Vmin and frame[row,col,2]<=Vmax)):count+=1row+=1if(count/height>0.6):left=colbreakcol+=1return top,button,left,rightdef recognize(img):top,button,left,right=locateAccuracy(img)image=img[top:button,left:right]if(image.size==0):return []img_gray=cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)img_thre=cv2.adaptiveThreshold(img_gray,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,35,2)#自适应二值化#对对图像进行垂直投影arr=np.zeros(image.shape[1])col=0while(col<img_thre.shape[1]):row=0count=0while(row<img_thre.shape[0]):count+=img_thre[row,col]row+=1arr[col]=int(count/255)col+=1#根据投影结果进行分割字符count_1=0flag=0flag_index=0i=0for c in arr:if(c<10):count_1+=1else:if(count_1>flag):flag=count_1flag_index=int(i-count_1/2)if(count_1>3):arr[int(i-count_1/2)]=-1count_1=0i+=1i=0j=0x=0y=toph=button-top#获得分割结果charList=[]for c in arr:if(c==-1):w=i-xcharList.append([x+left,y,w,h])x=iif(flag_index==c and (j!=1 or j!=2)):return []i+=1j+=1charList.append([x+left,y,right-x,h])if(len(charList)<=5 or len(charList)>8):return []return charListdef recognizeRect(img_thre,rect,knn):x,y,w,h=rectroi=img_thre[y:(y+h),x:(x+w)]if h>w:roi=cv2.copyMakeBorder(roi,0,0,int((h-w)/2),int((h-w)/2),cv2.BORDER_CONSTANT,value=[0,0,0])roi=cv2.resize(roi,(20,20))#cv2.imshow("char",roi)#cv2.waitKey(0)roivector=roi.reshape(1,400).astype(np.float32)ret,result,neighbours,dist=knn.findNearest(roivector,k=6)#进行预测return int(ret)def access_pixels(frame):frame = cv2.cvtColor(frame,cv2.COLOR_BGR2HSV)#print(frame.shape)  #shape内包含三个元素：按顺序为高、宽、通道数height = frame.shape[0]weight = frame.shape[1]count=0for row in range(height):            #遍历高for col in range(weight):         #遍历宽if((frame[row,col,0]>=100 and frame[row,col,0]<=124) and (frame[row,col,1]>=43 and frame[row,col,1]<=255) and (frame[row,col,2]>=46 and frame[row,col,2]<=255)):count+=1if(count/(height*weight)>0.5):return Trueelse:return Falsedef isPlate(frame):if(frame.shape[1]>frame.shape[0]/2 or frame.shape[1]*5<frame.shape[0]):return Trueelse:return Falsecap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640);
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480);print("Capture device is open: " + str(cap.isOpened()))period=20000
duty=6
pwm.generate(period,duty)time_start=time.time()
time_last=0
i=1
x_last=0
y_last=0
isRecognize=3
str_plate=""
print("start while")
while(cap.isOpened()):   if(base.buttons[0].read()==1):isRecognize=0elif(base.buttons[1].read()==1):isRecognize=1elif(base.buttons[2].read()==1):isRecognize=2elif(base.buttons[3].read()==1):isRecognize=3period=20000duty=6pwm.generate(period,duty)str_plate=""time_start=time.time()ret, frame = cap.read()if (ret):  if(isRecognize==3):outframe = hdmi_out.newframe()outframe[0:640,0:480,:] = frame[0:640,0:480,:]hdmi_out.writeframe(outframe)else:time_last=time_starttime_start=time.time()#print("read frame")image=cv2.resize(frame,(int(640*scale),400))#print("after new outframe time is:"+str(time.time()-time_start))img_gray=cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)#print("after cvtgray time is:"+str(time.time()-time_start))car_plates = cascade.detectMultiScale(img_gray, 1.1, 2, minSize=(36, 9), maxSize=(36 * 40, 9 * 40))#print("after mul time is:"+str(time.time()-time_start))for car_plate in car_plates:x, y, w, h = car_plate#plate = image[y: y + h, x: x + w]#if(isPlate(plate)==False):#根据颜色判断是否是正确的车牌区域#    continue#if(access_pixels(plate)==False):#根据颜色判断是否是正确的车牌区域#    continueif(isRecognize==0):plateScale=(w/PLATE_WIDTH)v=(math.sqrt((x-x_last)*(x-x_last)+(y-y_last)*(y-y_last))/(time_start-time_last)/plateScale)/iv=int(v*100)/100x_last=xy_last=yi=0cv2.putText(image,"Vehicle Velocity:"+str(v)+" m/s",(20,image.shape[0]-10),cv2.FONT_HERSHEY_PLAIN, 1.5, (0, 0, 255), 2)i+=1elif(isRecognize==1):if(base.buttons[0].read()==1):isRecognize=0elif(base.buttons[1].read()==1):isRecognize=1elif(base.buttons[2].read()==1):isRecognize=2elif(base.buttons[3].read()==1):isRecognize=3period=20000duty=6pwm.generate(period,duty)str_plate=""#分割字符img_thre=cv2.adaptiveThreshold(img_gray,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,35,2)#自适应二值化plate=image[y: y + h, x: x + w]charRect=recognize(image[y: y + h, x: x + w])if(len(charRect)==0):continueif(len(charRect)>7):charRect.pop()str_plate=""KeyCallBack();#模型创建knn_zh=cv2.ml.KNearest_create()knn_zh.train(train_data_zh,cv2.ml.ROW_SAMPLE,train_labels_zh)#识别中文rect=charRect[0]x1,y1,w1,h1=rectx1=x+x1y1=y+y1str_plate=labelDict[recognizeRect(img_thre,(x1,y1,w1,h1),knn_zh)]if(len(charRect)>0):x1,y1,w1,h1=charRect[0]cv2.rectangle(image,(x1,y1),(x1+w1,y1+h1),(255,0,0),1)if(base.buttons[0].read()==1):isRecognize=0elif(base.buttons[1].read()==1):isRecognize=1elif(base.buttons[2].read()==1):isRecognize=2elif(base.buttons[3].read()==1):isRecognize=3str_plate=""period=20000duty=6pwm.generate(period,duty)knn=cv2.ml.KNearest_create()knn.train(train_data,cv2.ml.ROW_SAMPLE,train_labels)for rect in charRect[1:]:x1,y1,w1,h1=rectx1=x+x1y1=y+y1cv2.rectangle(image,(x1,y1),(x1+w1,y1+h1),(255,0,0),1)#框出字块s=labelDict[recognizeRect(img_thre,(x1,y1,w1,h1),knn)]str_plate=str_plate+sfor re in reg:if(re in str_plate):period=20000duty=11pwm.generate(period,duty)print("********************ok")x1,y1,w1,h1=rectprint('recognize time:',time.time()-time_start,'s')image=cv2.putText(image,str_plate,(20,image.shape[0]-10),cv2.FONT_HERSHEY_PLAIN, 2.0, (0, 0, 255), 2)elif(isRecognize==2):image=cv2.putText(image,str_plate,(20,image.shape[0]-10),cv2.FONT_HERSHEY_PLAIN, 2.0, (0, 0, 255), 2)#标出粗定位的车牌cv2.rectangle(image, (x - 10, y - 10), (x + w + 10, y + h + 10), (255, 255, 255), 1)image=cv2.resize(image,(640,480))   outframe = hdmi_out.newframe()outframe[0:640,0:480,:] = image[0:640,0:480,:]hdmi_out.writeframe(outframe)#print("after write time is:"+str(time.time()-time_start))else:raise RuntimeError("Failed to read from camera.")
print("end program")