【爬虫】Selenium+chrome

一、下载对应chrome版本的webdriver

https://npm.taobao.org/mirrors/chromedriver

二、pom

<dependency>
   <groupId>org.seleniumhq.selenium</groupId>
   <artifactId>selenium-java</artifactId>
   <version>3.141.59</version>
</dependency>

三、初始化selenium

System.setProperty("webdriver.chrome.driver",driverPath+"/chromedriver.exe");
ChromeOptions chromeOptions=new ChromeOptions();
//设置 chrome 的无头模式
chromeOptions.setHeadless(Boolean.TRUE); //TRUE为无头模式
chromeOptions.addArguments("--disable-gpu"//禁止使用gpu
                ,"--blink-settings=imagesEnabled=false"//禁止加载图片
                ,"--user-agent=Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36"//使用user-agent
        );
chromeOptions.setCapability("enableVideo", false);//禁止播放视频
//启动一个 chrome 实例
webDriver = new ChromeDriver(chromeOptions);
webDriver.manage().timeouts().setScriptTimeout(15,TimeUnit.SECONDS);//设置15秒超时时间
//webDriver.manage().timeouts().implicitlyWait(15,TimeUnit.SECONDS);
webDriver.manage().timeouts().pageLoadTimeout(15,TimeUnit.SECONDS);

四、访问URL，返回页面dom

try {
     webDriver.get(url);
}catch (TimeoutException e){
     ((JavascriptExecutor)webDriver).executeScript("window.stop()");
}
return webDriver.getPageSource();

五、输入框填值

WebDriverWait wait1 = new WebDriverWait(seleniumUtil.getWebDriver(), 10);
WebElement element1=wait1.until(ExpectedConditions.elementToBeClickable(By.cssSelector("input[name=username]")));
element1.click();
element1.sendKeys(userName);