Selenium+Tesseract-OCR智能识别验证码爬取网页数据

1.项目需求描述

通过订单号获取某系统内订单的详细数据，不需要账号密码的登录验证，但有图片验证码的动态识别，将获取到的数据存到数据库。

2.整体思路

1.通过Selenium技术，无窗口模式打开浏览器

2.在输入框中动态输入订单号

3.将图片验证码截图保存到本地

4.通过Tesseract-OCR技术去本地识别验证码转化为文字

5.将获取的验证码输入输入框

6.点击查询获取列表数据

3.功能实现

1.下载并安装Google浏览器，安装Google驱动chromedriver.exe，获取安装路径，配置在项目中

2.使用Selenium进行浏览器操作

 1 System.setProperty(浏览器驱动, 浏览器驱动安装位置);
 2 ChromeOptions options = new ChromeOptions();
 3 options.addArguments("--headless");                            // 无窗口模式
 4 options.addArguments("--disable-infobars");                    // 禁言消息条
 5 options.addArguments("--disable-extensions");                  // 禁用插件
 6 options.addArguments("--disable-gpu");                         // 禁用GPU
 7 options.addArguments("--no-sandbox");                          // 禁用沙盒模式
 8 options.addArguments("--disable-dev-shm-usage");
 9 options.addArguments("--hide-scrollbars");                     // 隐藏滚动条
10
11 WebDriver driver = new ChromeDriver(options);
12 driver.get(爬取网站URL);
13 driver.manage().window().setSize(new Dimension(450, 260));     // 设置游览器打开后调整大小
14 try {
15     // 保存IMG图片到本地
16     saveImgToLocal(driver);
17     Thread.sleep(2000);
18     // OCR智能识别验证码
19     String codeByOCR = getCodeByOCR();
20     if (codeByOCR != null) {
21         try {
22             WebElement input1 = driver.findElement(By.id(TEXTBOX1));
23             input1.sendKeys(code);
24             WebElement input2 = driver.findElement(By.id(TEXTBOX2));
25             input2.sendKeys(codeByOCR);
26             // 获取table数据
27             WebElement addButton = driver.findElement(By.id(SELECT_BUTTON));
28             addButton.click();
29             List<WebElement> tRCollection = driver.findElement(By.id(TABLE_ID)).findElements(By.tagName("tr"));
30             for (int t = 1; t < tRCollection.size(); t++) {
31                 List<WebElement> tDCollection = tRCollection.get(t).findElements(By.tagName("td"));
32                 VipLogisticsMinHangDetailVo minHangDetailVo = new VipLogisticsMinHangDetailVo();
33                 minHangDetailVo.setLogistics_number(code);
34                 for (int i = 0; i < tDCollection.size(); i++) {
35                     String text = tDCollection.get(i).getText();
36                     switch (i) {
37                         case 0:
38                             minHangDetailVo.setTime(text);
39                         case 1:
40                             minHangDetailVo.setOutlet(text);
41                         case 2:
42                             minHangDetailVo.setOrganization(text);
43                         case 3:
44                             minHangDetailVo.setEvent(text);
45                         case 4:
46                             minHangDetailVo.setDetail(text);
47                     }
48                 }
49                 list.add(minHangDetailVo);
50             }
51             log.info("验证码识别成功！");
52         } catch (Exception e) {
53             if (e.toString().contains("错误提示：验证码错误或已过期！")) {
54                 log.error("验证码识别错误！" + e.toString());
55             } else if (e.toString().contains("错误提示：请输入验证码！")) {
56                 log.error("未输入验证码！：" + e.toString());
57             } else {
58                 log.error("其他异常：" + e.toString());
59             }
60         }
61     }
62     driver.quit();
63 } catch (Exception e) {
64     e.printStackTrace();
65 }

3.将图片验证码截图保存到本地（截屏法）

 1 private void saveImgToLocal(WebDriver driver) {
 2     WebElement element = driver.findElement(By.id(img元素ID));
 3     //创建全屏截图
 4     WrapsDriver wrapsDriver = (WrapsDriver) element;
 5     File screen = ((TakesScreenshot) wrapsDriver.getWrappedDriver()).getScreenshotAs(OutputType.FILE);
 6     try {
 7         BufferedImage image = ImageIO.read(screen);
 8         //创建一个矩形使用上面的高度，和宽度
 9         Point p = element.getLocation();
10         //元素坐标
11         BufferedImage img = image.getSubimage(p.getX(), p.getY(), element.getSize().getWidth(), element.getSize().getHeight());
12         ImageIO.write(img, "png", screen);
13
14         FileUtils.copyFile(screen, new File(保存本地地址 + "imgname.png"));
15     } catch (IOException e) {
16         e.printStackTrace();
17     }
18 }

4.将图片验证码保存到本地（鼠标法）

 1 private static void saveImgToLocal1(WebDriver driver) {
 2     Actions action = new Actions(driver);
 3     action.contextClick(driver.findElement(By.id(img元素ID))).build().perform();
 4     try {
 5         Robot robot = new Robot();
 6         Thread.sleep(1000);
 7
 8         robot.keyPress(KeyEvent.VK_DOWN);
 9         Thread.sleep(1000);
10
11         robot.keyPress(KeyEvent.VK_DOWN);
12         Thread.sleep(1000);
13
14         robot.keyPress(KeyEvent.VK_ENTER);
15         Thread.sleep(1000);
16         //释放向下键，不然在此之前的条目将起作用
17         robot.keyRelease(KeyEvent.VK_DOWN);
18         Thread.sleep(1000);
19         //运行保存
20         Runtime.getRuntime().exec(SAVE_IMG_EXE);
21         Thread.sleep(10000);
22     } catch (Exception e) {
23         e.printStackTrace();
24     }
25 }

 1 private static void saveImgToLocal1(WebDriver driver) {
 2     Actions action = new Actions(driver);
 3     action.contextClick(driver.findElement(By.id(img元素ID))).build().perform();
 4     try {
 5         Robot robot = new Robot();
 6         Thread.sleep(1000);
 7
 8         robot.keyPress(KeyEvent.VK_DOWN);
 9         Thread.sleep(1000);
10
11         robot.keyPress(KeyEvent.VK_DOWN);
12         Thread.sleep(1000);
13
14         robot.keyPress(KeyEvent.VK_ENTER);
15         Thread.sleep(1000);
16         //释放向下键，不然在此之前的条目将起作用
17         robot.keyRelease(KeyEvent.VK_DOWN);
18         Thread.sleep(1000);
19         //运行保存
20         Runtime.getRuntime().exec(SAVE_IMG_EXE);
21         Thread.sleep(10000);
22     } catch (Exception e) {
23         e.printStackTrace();
24     }
25 }

5.对本地验证码进行OCR识别

 1 private String getCodeByOCR() {
 2     String result = null;
 3     File file = new File(本地图片地址);
 4     if (!file.exists()) {
 5         if (systemFalg != 1) {
 6             file.setWritable(true, false);
 7         }
 8         file.mkdirs();
 9     }
10     File imageFile = new File(本地图片地址 + "imgname.png");
11     if (imageFile.exists()) {
12         ITesseract instance = new Tesseract();
13         instance.setDatapath(tessdata存放地址);
14         try {
15             String doOCR = instance.doOCR(imageFile);
16             result = replaceBlank(doOCR);
17             log.info("解析的验证码为:{}", result != null ? result : "为空！");
18         } catch (Exception e) {
19             e.printStackTrace();
20             log.error("解析验证码异常！");
21         }
22     } else {
23         log.error("解析验证码的文件不存在！");
24     }
25     return result;
26 }

综上，该网页的数据就可以获取了。