java爬虫实现爬取百度风云榜Top10

最近在项目中遇到了java和python爬虫进行程序调用和接口对接的问题, 刚开始也是调试了好久才得出点门道.

而后,自己也发现了爬虫的好玩之处,边想着用java来写个爬虫玩玩,虽说是个不起眼的demo,但还是想记录一下这个小爬虫,便于以后的查阅.

直接上代码:

 import org.jsoup.Connection;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
 import org.springframework.util.StringUtils;

 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;

 public class MySpider {
     public static void main(String[] args) {
         List<NewsEntity> list = new ArrayList<NewsEntity>();
         Connection connect = Jsoup.connect("http://top.baidu.com/buzz?b=1&fr=tph_right");  //百度风云榜网址
         connect.userAgent("Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)");  //模拟火狐浏览器访问网页
         try {
             Document document = connect.get();      //建立连接,获取网页内容为文档对象
             Element main = document.getElementById("main");  //获取需要爬去部位的根元素
             Elements url = main.select("div[class=mainBody]").select("table[class=list-table]")
                     .select("tbody").select("tr"); //css选择器
             int i = 0;
             for (Element element : url) {
                 NewsEntity entity = new NewsEntity();
                 String attr_url = element.select("td[class=keyword]").select("a[class=list-title]").attr("href");
                 String text = element.select("td[class=keyword]").select("a[class=list-title]").text();
                 String span = element.select("td[class=last").select("span").text();
                 if (StringUtils.isEmpty(attr_url) || StringUtils.isEmpty(text) || StringUtils.isEmpty(span)) {
                     continue;
                 }
                 entity.setTitle(text);
                 entity.setUrl(attr_url);
                 entity.setHots(span);
                 i++;
                 if (i > 10) {
                     break;
                 }
                 list.add(entity);

             }
             System.out.println(list.toString());
             System.out.println(list.size());

         } catch (IOException e) {
             e.printStackTrace();
             System.out.println("网页元素发生改变或访问被禁止");
         }
     }
 }

简易封装:

 /**
  * @author RYH
  * @description 封装新闻实体
  * @date 2019/2/26
  **/
 public class NewsEntity {
     private String title;
     private String url;
     private String hots;

     public String getTitle() {
         return title;
     }

     public void setTitle(String title) {
         this.title = title;
     }

     public String getUrl() {
         return url;
     }

     public void setUrl(String url) {
         this.url = url;
     }

     public String getHots() {
         return hots;
     }

     public void setHots(String hots) {
         this.hots = hots;
     }

     @Override
     public String toString() {
         return "NewsEntity{" +
                 "title='" + title + '\'' +
                 ", url='" + url + '\'' +
                 ", hots=" + hots +
                 '}';
     }
 }

导入的包也只有jsoup包,功能还是很强大的

<dependency>
　　<groupId>org.jsoup</groupId>
　　<artifactId>jsoup</artifactId>
　　<version>1.12.1</version>
</dependency>

控制台打印也一目了然, 做些简单的爬取还是很容易的