opencv图片处理与OCR识别-白红宇

经过一个月的研究、opencv、能处理图片，并半吊子识别。暂时还是没有数据，不能实现需求护照识别，对身份证的识别，又因为中文的原因，识别率不高。其次针对护照处理图片的参数需要动态配置。

对于只熟悉java的开发，查找资料学习opencv有些困难，网上大多都是C++版本的，并且，2.4，3.4库有改动。记录一下opencv的基本操作，一起学习。并提供一种ocr思路

一、处理身份证案例：

Opencv获取身份证号码区域的示例代码

1、对图片进行降噪以及二值化，凸显内容区域2、对图片进行轮廓检测3、对轮廓结果进行分析4、剪裁指定区域复制代码

处理过程

灰度图

高斯模糊降噪 GaussianBlur

二值化 threshold

中值滤波降噪 medianBlur

腐蚀操作 erode

二、一般处理代码

- 2.4版本//获取图片Mat templateImage = Highgui.imread(templateFilePath, Highgui.CV_LOAD_IMAGE_COLOR);//灰度Imgproc.cvtColor(img2, img2,  Imgproc.COLOR_BGR2GRAY);//高斯滤波Imgproc.GaussianBlur(img2, img2, new Size(3,3), 0);//中值滤波Imgproc.medianBlur(img2,img2,3);//腐蚀Imgproc.erode(originalImage, originalImage, new Mat(14, 14, 0));//可调节阈值二值Imgproc.adaptiveThreshold(img2, img2, 255, Imgproc.ADAPTIVE_THRESH_GAUSSIAN_C, Imgproc.THRESH_BINARY, 33, 25);//只过滤黑色Mat imgHSV = new Mat(img.rows(), img.cols(), CvType.CV_8UC3);Imgproc.cvtColor(img, imgHSV, Imgproc.COLOR_BGR2GRAY);Scalar minValues = new Scalar(0, 0, 0);Scalar maxValues = new Scalar(107, 107, 107);Mat mask = new Mat();Core.inRange(imgHSV, minValues, maxValues, mask);//边缘检测矩形识别，并标注 List
    
      contours=new ArrayList<>();Mat mat=new Mat();Imgproc.findContours(originalImage1,contours,mat,Imgproc.RETR_LIST,Imgproc.CHAIN_APPROX_NONE);Mat originalImage12=originalImage1;for (int i = 0; i < contours.size(); i++) {    Rect rect = Imgproc.boundingRect(contours.get(i));    Core.rectangle(originalImage12, rect.tl(), rect.br(), new Scalar(255, 0, 255));    originalImage12.submat(rect);}Highgui.imwrite(Imginfo.PATH_CACHE+UUID.randomUUID().toString()+Imginfo.JPG_SUFFIX, originalImage12);复制代码

三、图形匹配，仿射变换、旋转

参考链接

package com.cyd.ocr.passportocr;import com.sun.image.codec.jpeg.JPEGCodec;import com.sun.image.codec.jpeg.JPEGEncodeParam;import com.sun.image.codec.jpeg.JPEGImageEncoder;import org.opencv.calib3d.Calib3d;import org.opencv.core.*;import org.opencv.features2d.*;import org.opencv.highgui.Highgui;import org.opencv.imgproc.Imgproc;import javax.imageio.ImageIO;import java.awt.image.BufferedImage;import java.io.*;import java.util.LinkedList;import java.util.List;/** * * 根据特征点，匹配模板，旋转截取，根据比例特定区域进行识别。 * * 未改进点：只过滤出黑色进行识别。 * @author chenyd * @date 2018/10/7 18:34 */public class test_ocr {    public static int DPI = 300;    private float nndrRatio = 0.7f;//这里设置既定值为0.7，该值可自行调整    private int matchesPointCount = 0;    public float getNndrRatio() {        return nndrRatio;    }    public void setNndrRatio(float nndrRatio) {        this.nndrRatio = nndrRatio;    }    public int getMatchesPointCount() {        return matchesPointCount;    }    public void setMatchesPointCount(int matchesPointCount) {        this.matchesPointCount = matchesPointCount;    }    public void matchImage(Mat templateImage, Mat originalImage) {        MatOfKeyPoint templateKeyPoints = new MatOfKeyPoint();        //指定特征点算法SURF        FeatureDetector featureDetector = FeatureDetector.create(FeatureDetector.SURF);        //获取模板图的特征点        featureDetector.detect(templateImage, templateKeyPoints);        //提取模板图的特征点        MatOfKeyPoint templateDescriptors = new MatOfKeyPoint();        DescriptorExtractor descriptorExtractor = DescriptorExtractor.create(DescriptorExtractor.SURF);        System.out.println("提取模板图的特征点");        descriptorExtractor.compute(templateImage, templateKeyPoints, templateDescriptors);        //显示模板图的特征点图片        Mat outputImage = new Mat(templateImage.rows(), templateImage.cols(), Highgui.CV_LOAD_IMAGE_COLOR);        System.out.println("在图片上显示提取的特征点");        Features2d.drawKeypoints(templateImage, templateKeyPoints, outputImage, new Scalar(255, 0, 0), 0);        //获取原图的特征点        MatOfKeyPoint originalDescriptors = new MatOfKeyPoint();        MatOfKeyPoint originalKeyPoints = new MatOfKeyPoint();        featureDetector.detect(originalImage, originalKeyPoints);        System.out.println("提取原图的特征点");        descriptorExtractor.compute(originalImage, originalKeyPoints, originalDescriptors);        List
    
      matches = new LinkedList();        DescriptorMatcher descriptorMatcher = DescriptorMatcher.create(DescriptorMatcher.FLANNBASED);        System.out.println("寻找最佳匹配");        /**         * knnMatch方法的作用就是在给定特征描述集合中寻找最佳匹配         * 使用KNN-matching算法，令K=2，则每个match得到两个最接近的descriptor，然后计算最接近距离和次接近距离之间的比值，当比值大于既定值时，才作为最终match。         */        descriptorMatcher.knnMatch(templateDescriptors, originalDescriptors, matches, 2);        System.out.println("计算匹配结果");        LinkedList
     
       goodMatchesList = new LinkedList();        //对匹配结果进行筛选，依据distance进行筛选        matches.forEach(match -> {            DMatch[] dmatcharray = match.toArray();            DMatch m1 = dmatcharray[0];            DMatch m2 = dmatcharray[1];            if (m1.distance <= m2.distance * nndrRatio) {                goodMatchesList.addLast(m1);            }        });        matchesPointCount = goodMatchesList.size();        //当匹配后的特征点大于等于 4 个，则认为模板图在原图中，该值可以自行调整        if (matchesPointCount >= 4) {            System.out.println("模板图在原图匹配成功！");            List
      
        templateKeyPointList = templateKeyPoints.toList();            List
       
         originalKeyPointList = originalKeyPoints.toList();            LinkedList
        
          objectPoints = new LinkedList();            LinkedList
         
           scenePoints = new LinkedList(); goodMatchesList.forEach(goodMatch -> { objectPoints.addLast(templateKeyPointList.get(goodMatch.queryIdx).pt); scenePoints.addLast(originalKeyPointList.get(goodMatch.trainIdx).pt); }); MatOfPoint2f objMatOfPoint2f = new MatOfPoint2f(); objMatOfPoint2f.fromList(objectPoints); MatOfPoint2f scnMatOfPoint2f = new MatOfPoint2f(); scnMatOfPoint2f.fromList(scenePoints); //使用 findHomography 寻找匹配上的关键点的变换 Mat homography = Calib3d.findHomography(objMatOfPoint2f, scnMatOfPoint2f, Calib3d.RANSAC, 3); /** * 透视变换(Perspective Transformation)是将图片投影到一个新的视平面(Viewing Plane)，也称作投影映射(Projective Mapping)。 */ Mat templateCorners = new Mat(4, 1, CvType.CV_32FC2); Mat templateTransformResult = new Mat(4, 1, CvType.CV_32FC2); templateCorners.put(0, 0, new double[]{0, 0}); templateCorners.put(1, 0, new double[]{templateImage.cols(), 0}); templateCorners.put(2, 0, new double[]{templateImage.cols(), templateImage.rows()}); templateCorners.put(3, 0, new double[]{0, templateImage.rows()}); //使用 perspectiveTransform 将模板图进行透视变以矫正图象得到标准图片 Core.perspectiveTransform(templateCorners, templateTransformResult, homography); //矩形四个顶点 double[] pointA = templateTransformResult.get(0, 0); double[] pointB = templateTransformResult.get(1, 0); double[] pointC = templateTransformResult.get(2, 0); double[] pointD = templateTransformResult.get(3, 0);// System.out.println(String.format("【%s，%s】,【%s，%s】,【%s，%s】,【%s，%s】",pointA[0],pointA[1],pointB[0],pointB[1], pointC[0],pointC[1],pointD[0],pointD[1])); //左上，右上点之间的距离 double range=getDistance(new Point(pointA),new Point(pointB)); double sina=Math.abs(pointA[1]-pointB[1]); double jd=Math.asin(sina/range)/Math.PI*180; System.out.println("旋转角度："+jd); Mat jdmat=rotate3(originalImage,-jd); String xz="C:\\Users\\chenyd\\Desktop\\img\\idcode\\jdmat.jpg"; Highgui.imwrite(xz, jdmat); if(jd > 1){ System.out.println("匹配旋转之后的图片"); matchImage(templateImage,jdmat); return; } //指定取得数组子集的范围 int rowStart = (int) pointA[1]; int rowEnd = (int) pointC[1]; int colStart = (int) pointD[0]; int colEnd = (int) pointB[0]; int temp=0; if(rowStart>rowEnd){ temp=rowStart; rowStart=rowEnd; rowEnd=temp; } if(colStart>colEnd){ temp=colStart; colStart=colEnd; colEnd=temp; } //TODO 大于零 System.out.println(String.format("%s,%s,%s,%s",rowStart, rowEnd, colStart, colEnd)); Mat subMat = originalImage.submat(rowStart, rowEnd, colStart, colEnd); Highgui.imwrite("C:\\Users\\chenyd\\Desktop\\img\\idcode\\match.jpg", subMat); subTarget(subMat); //将匹配的图像用用四条线框出来 Core.line(originalImage, new Point(pointA), new Point(pointB), new Scalar(0, 255, 0), 4);//上 A->B Core.line(originalImage, new Point(pointB), new Point(pointC), new Scalar(0, 255, 0), 4);//右 B->C Core.line(originalImage, new Point(pointC), new Point(pointD), new Scalar(0, 255, 0), 4);//下 C->D Core.line(originalImage, new Point(pointD), new Point(pointA), new Scalar(0, 255, 0), 4);//左 D->A MatOfDMatch goodMatches = new MatOfDMatch(); goodMatches.fromList(goodMatchesList); Mat matchOutput = new Mat(originalImage.rows() * 2, originalImage.cols() * 2, Highgui.CV_LOAD_IMAGE_COLOR); Features2d.drawMatches(templateImage, templateKeyPoints, originalImage, originalKeyPoints, goodMatches, matchOutput, new Scalar(0, 255, 0), new Scalar(255, 0, 0), new MatOfByte(), 2); Highgui.imwrite("C:\\Users\\chenyd\\Desktop\\img\\idcode\\matchOutput.jpg", matchOutput); Highgui.imwrite("C:\\Users\\chenyd\\Desktop\\img\\idcode\\originalImage.jpg", originalImage); } else { System.out.println("模板图不在原图中！"); } Highgui.imwrite("C:\\Users\\chenyd\\Desktop\\img\\idcode\\outputImage.jpg", outputImage); } private void subTarget(Mat originalImage) { //Imgproc.GaussianBlur(originalImage, originalImage, new Size(3,3), 0); //Imgproc.medianBlur(originalImage,originalImage,3); Imgproc.cvtColor(originalImage, originalImage, Imgproc.COLOR_BGR2GRAY); Imgproc.adaptiveThreshold(originalImage, originalImage, 255, Imgproc.ADAPTIVE_THRESH_MEAN_C, Imgproc.THRESH_BINARY, 45, 55); Highgui.imwrite("C:\\Users\\chenyd\\Desktop\\img\\idcode\\medianBlurMat.jpg", originalImage); int imgrow=originalImage.rows(); int imgcol=originalImage.cols(); double[] tarT={930,550};//模板中A点坐标 //PR// double[] tarA={301,108};// double[] tarC={365,138}; //NAME double[] tarA={307,163}; double[] tarC={492,193}; //DATE// double[] tarA={300,422};// double[] tarC={660,460}; /* double[] tarT={1054,553}; //NAME double[] tarA={314,173}; double[] tarC={607,216}; //NAME// double[] tarA={515,116};// double[] tarC={680,145};*/ //识别区左上和右下的相对距离 int targRowS=(int) (tarA[1]/tarT[1]*imgrow);//列 int targRowE=(int) (tarC[1]/tarT[1]*imgrow); int targColS=(int) (tarA[0]/tarT[0]*imgcol);//行 int targColE=(int) (tarC[0]/tarT[0]*imgcol); Mat subMat1 = originalImage.submat( targRowS,targRowE, targColS, targColE); String file ="C:\\Users\\chenyd\\Desktop\\img\\idcode\\subTarget.jpg"; Highgui.imwrite(file, subMat1); //BufferedImage bi=Mat2Img(subMat1,".jpg"); handleDpi( new File(file), DPI, DPI); tesseract(file); } public static void handleDpi(File file, int xDensity, int yDensity) { FileOutputStream out=null; try { BufferedImage image = ImageIO.read(file); out=new FileOutputStream(file); JPEGImageEncoder jpegEncoder = JPEGCodec.createJPEGEncoder(out); JPEGEncodeParam jpegEncodeParam = jpegEncoder.getDefaultJPEGEncodeParam(image); jpegEncodeParam.setDensityUnit(JPEGEncodeParam.DENSITY_UNIT_DOTS_INCH); jpegEncoder.setJPEGEncodeParam(jpegEncodeParam); //jpegEncodeParam.setQuality(0.75f, false); //jpegEncodeParam.setQuality(2f, false); jpegEncodeParam.setXDensity(xDensity); jpegEncodeParam.setYDensity(yDensity); jpegEncoder.encode(image, jpegEncodeParam); image.flush(); } catch (IOException e) { e.printStackTrace(); }finally { if(out!=null){ try { out.close(); } catch (IOException e) { e.printStackTrace(); } } } } public double getDistance(Point p,Point p2){ double _x = Math.abs(p.x - p2.x); double _y = Math.abs(p.y - p2.y); return Math.sqrt(_x*_x+_y*_y); } public static Mat rotate3(Mat splitImage, double angle) { double thera = angle * Math.PI / 180; double a = Math.sin(thera); double b = Math.cos(thera); int wsrc = splitImage.width(); int hsrc = splitImage.height(); int wdst = (int) (hsrc * Math.abs(a) + wsrc * Math.abs(b)); int hdst = (int) (wsrc * Math.abs(a) + hsrc * Math.abs(b)); Mat imgDst = new Mat(hdst, wdst, splitImage.type()); Point pt = new Point(splitImage.cols() / 2, splitImage.rows() / 2); // 获取仿射变换矩阵 Mat affineTrans = Imgproc.getRotationMatrix2D(pt, angle, 1.0); // 改变变换矩阵第三列的值 affineTrans.put(0, 2, affineTrans.get(0, 2)[0] + (wdst - wsrc) / 2); affineTrans.put(1, 2, affineTrans.get(1, 2)[0] + (hdst - hsrc) / 2); Imgproc.warpAffine(splitImage, imgDst, affineTrans, imgDst.size()); return imgDst; } public static BufferedImage Mat2Img(Mat mat, String fileExtension) { MatOfByte mob = new MatOfByte(); Highgui.imencode(fileExtension, mat, mob); byte[] byteArray = mob.toArray(); BufferedImage bufImage = null; try { InputStream in = new ByteArrayInputStream(byteArray); bufImage = ImageIO.read(in); } catch (Exception e) { e.printStackTrace(); } return bufImage; } public String tesseract(String file1) { String result = ""; String imgPath=file1; BufferedReader bufReader = null; try { String outPath = imgPath.substring(0, imgPath.lastIndexOf(".")); Runtime runtime = Runtime.getRuntime(); String command = "tesseract" + " " + imgPath + " " + outPath +" -l eng --psm 7 "; System.out.println(command); Process ps = runtime.exec(command); ps.waitFor(); // 读取文件 File file = new File(outPath + ".txt"); bufReader = new BufferedReader(new FileReader(file)); String temp = ""; StringBuffer sb = new StringBuffer(); while ((temp = bufReader.readLine()) != null) { sb.append(temp); } // 文字结果 result = sb.toString(); //if (!StringUtils.isEmpty(result)) // result = result.replaceAll(" ", ""); System.out.println("识别结果>>>>>>>>>: "+result); } catch (Exception e) { e.printStackTrace(); } return result; } public static void imshow(Mat image, String windowName){// try {// UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());// } catch (ClassNotFoundException e) {// e.printStackTrace();// } catch (InstantiationException e) {// e.printStackTrace();// } catch (IllegalAccessException e) {// e.printStackTrace();// } catch (UnsupportedLookAndFeelException e) {// e.printStackTrace();// }//// JFrame jFrame = new JFrame(windowName);// JLabel imageView = new JLabel();// final JScrollPane imageScrollPane = new JScrollPane(imageView);// imageScrollPane.setPreferredSize(new Dimension(500, 500)); // set window size// jFrame.add(imageScrollPane, BorderLayout.CENTER);// jFrame.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE);//// Image loadedImage = Mat2BufferedImage(image);// imageView.setIcon(new ImageIcon(loadedImage));// jFrame.pack();// jFrame.setLocationRelativeTo(null);// jFrame.setVisible(true); } public static void main(String[] args) { System.loadLibrary(Core.NATIVE_LIBRARY_NAME);// String templateFilePath = "C:\\Users\\chenyd\\Desktop\\img\\idcode\\gbc_1.jpg";// String originalFilePath = "C:\\Users\\chenyd\\Desktop\\img\\idcode\\GRC.jpg"; String templateFilePath = "C:\\Users\\chenyd\\Desktop\\img\\idcode\\bhs_4.jpg"; String originalFilePath = "C:\\Users\\chenyd\\Desktop\\img\\idcode\\1539152678458.jpg";// String originalFilePath = "C:\\Users\\chenyd\\Desktop\\img\\idcode\\BHS_3.jpg";// String templateFilePath = "C:\\Users\\chenyd\\Desktop\\img\\idcode\\SGP_M_1.jpg";// String originalFilePath = "C:\\Users\\chenyd\\Desktop\\img\\idcode\\SGP_.jpg"; //读取图片文件 Mat templateImage = Highgui.imread(templateFilePath, Highgui.CV_LOAD_IMAGE_COLOR); Mat originalImage = Highgui.imread(originalFilePath, Highgui.CV_LOAD_IMAGE_COLOR); test_ocr imageRecognition = new test_ocr(); imageRecognition.matchImage(templateImage, originalImage); System.out.println("匹配的像素点总数：" + imageRecognition.getMatchesPointCount()); }}复制代码

四、自定义图像识别思路

识别

识别文字使用开源技术：tesseract 中文识别率不高，需要下载chi文件，可对中文进行训练

图片处理

使用opencv，大多数都是C++语言，支持java 2.4版本可以使用图片匹配，3.4版本不行，报错

opencv处理图片，之后由tesseract识别。

opencv处理流程

将需要识别的图片制作成模板，扣去需要识别的信息，保留不变的元素。

获取base64图片。

与模板匹配、并扣取匹配的原图。

将匹配的原图，过滤黑色，只保留文字

将过滤之后的图，矩形匹配，并增加矩形过滤规则，保留符合要求的矩形

订制模板中信息的大概矩形位置，矩形匹配大致都落入到模板信息区域，得到目标矩形坐标

将扣取的原图进行灰度、滤波、中值、二值化、并截取目标矩形坐标。