爬虫（Java实现）

说明：

　　使用了htmlparser库。

　　运行过程：

　　　　从某个网址开始，摘取网页中的链接，并通过广度搜索，对这些链接递归执行上述操作。

　　　　在以上过程中把网址存入数据库中。以防止搜索中出现环路。

　　　　但是，程序经常进入某个网站后，会一直扫描其中的二级域名。

　　　　于是数据库中会出现这种情况：

　　　　　　jack.art.com

　　　　　　han.art.com

　　　　　　bob.art.com

　　　　　　alice.art.com

　　　　　　rose.art.com

　　　　　　...

　　　　　　...

代码：

//Robot.java  1 package robot;
 import java.net.*;
 import java.sql.SQLException;
 import java.util.Random;

 import javax.swing.JOptionPane;

 import org.htmlparser.*;
 import org.htmlparser.filters.TagNameFilter;
 import org.htmlparser.tags.LinkTag;
 import org.htmlparser.util.NodeList;
 import org.htmlparser.util.ParserException;

 import mydb.DB;
 public class Robot {
     int ff=0;

     int num=0;
     //DB db;
     Robot() throws MalformedURLException, SQLException{
         DB.getConnect("localhost","3306","robot","root","142365");
         DB.getSta();
         String url0="http://www.youku.com";//"http://localhost";
      /*  DB.rs= DB.s.executeQuery("select count(*) from urls");
         if(DB.rs.next())
         {int n=DB.rs.getInt(1);
         System.out.println(n );
         Random random = new Random();

         int ran = random.nextInt();
         ran%=n;
         ran=ran>0?ran:-ran;
         System.out.println(ran );
         DB.rs= DB.s.executeQuery("select * from urls");
         int x=0;
         while(DB.rs.next()&&x<ran){

                   System.out.println(DB.rs.getString(1)+"000" ); url0=DB.rs.getString(1);

               x++;
         }

         }*/
         //catchHref("http://localhost",num);
         catchHref(url0,num);
     }
     boolean isEndLegal(String str){
         if(str.endsWith("php")||str.endsWith("net/")||str.endsWith("com/")||str.endsWith("cn/")||str.endsWith("gov/")||str.endsWith("edu/")||str.endsWith("org/")||str.endsWith("net")||str.endsWith("com")||str.endsWith("cn")||str.endsWith("gov")||str.endsWith("edu")||str.endsWith("org")){
             return true;
         }
         return false;
     }
     boolean catchHref(String hreft ,int num) throws MalformedURLException {
         Parser     parser =null;
         NodeList nodelist=null;
         String href = "http://www.baidu.com";
         //db=new DB();
         if(ff!=0)
         if (!(hreft.startsWith("http")&&isEndLegal(hreft)&&!isInDatabase(hreft))) {
             return false;
         }
         ff=1;
         add(hreft);

             System.out.println(num);
             try {
                 parser = new Parser(hreft);
                 if(parser==null)return false;
             } catch (ParserException e) {
                 return false;
                 //e.printStackTrace();
             }
             try {
                 nodelist = parser.parse(null);
             } catch (ParserException e1) {
                 e1.printStackTrace();
             }
             if(nodelist==null)return false;
             NodeFilter filter = new TagNameFilter("A");
             if(filter==null)return false;
             nodelist = nodelist.extractAllNodesThatMatch(filter, true);
             if(nodelist==null)return false;
             for (int i = 0; i < nodelist.size(); i++) {
                 LinkTag link = (LinkTag) nodelist.elementAt(i);
                 href = link.getAttribute("href");
                 if(href==null)return false;
                 System.out.println(href );
                 catchHref(href,num);

             }
             num++;
         return true;
     }
 void add(String str){

         try {
             DB.s.execute("insert into urls2(url)values('"+str+"');");
             DB.commit();
             System.out.println("add");
         } catch (SQLException e) {
             //e.printStackTrace();return ;
             //JOptionPane.showMessageDialog(null, "数据库添加失败");
             //System.exit(-1);

         }

         return ;
     }
     boolean isInDatabase(String str){

         try {
             DB.rs= DB.s.executeQuery("select * from urls where url like'"+str+"%';");
             if(DB.rs.next()){System.out.println(DB.rs);return true;}
         } catch (SQLException e) {
             e.printStackTrace();
             JOptionPane.showMessageDialog(null, "数据库查找失败");
             System.exit(-1);
         }
         return false;
     }
     public static void main(String[] args)
             throws MalformedURLException, ParserException, SQLException {
         Robot robot = new Robot();
     }
 }

//DB.java  1 package mydb;
 import java.sql.*;
 import java.util.ArrayList;

 import javax.swing.*;
 //import com.mysql.jdbc.Driver;
 public class DB {

     public static Connection conn = null;
     public static ResultSet rs = null;
     public static Statement s = null;
     public DB() {
         conn = null;
         s = null;
         rs=null;

     }
     /*
     String getResult(ResultSet rs) {
         String str = "Book\t\tOwnerID\tOwnerName\n";
         // System.out.println("\nno\tname\tsex\tsalary");
         try {
             while (rs.next()) {
                 StringBuilder builder = new StringBuilder(rs.getString(1));
                 builder.append("\t\t");
                 builder.append(rs.getString(2));
                 builder.append("\t");
                 builder.append(rs.getString(3));
                 builder.append("\n");
                 str += builder.toString();
             }
         } catch (Throwable e) {

         }
         // System.out.println();
         return str;
     }*/

     public static Connection getConnect(String IP,String port,String database,String user,String password){
         try {
             // Class.forName("org.gjt.mm.mysql.Driver").newInstance();
             Class.forName("com.mysql.jdbc.Driver");
         } catch (ClassNotFoundException e1) {
             e1.printStackTrace();
             JOptionPane.showMessageDialog(null, "数据库包未找到");
             System.exit(-1);
         } // .newInstance();

         try {
             conn = DriverManager.getConnection(
                     "jdbc:mysql://"+IP+":"+port+"/"+database+"?useUnicode=true&characterEncoding=utf8", user,
                     password);//autoReconnect=true&useUnicode=true&characterEncoding=utf8
         } catch (SQLException e1) {
             e1.printStackTrace();
             JOptionPane.showMessageDialog(null, "数据库无法连接");
             System.exit(-1);
         }

         try {
             conn.setAutoCommit(false);
         } catch (SQLException e1) {

             e1.printStackTrace();
         }

         return conn;
     }
     public static Statement getSta(){
         try {
             s = conn.createStatement();
         } catch (SQLException e1) {

             e1.printStackTrace();
             JOptionPane.showMessageDialog(null, "无法建立数据库语句");
             System.exit(-1);
         }
         return s;
     }

     public    static int commit(){
         try {
             conn.commit();
         } catch (SQLException e) {

             e.printStackTrace();
             JOptionPane.showMessageDialog(null, "对数据库更改无法应用");

         }
         return 0;
     }
     public static void closeConnect() {
         try {
             rs.close();
         } catch (SQLException e) {
             e.printStackTrace();
             JOptionPane.showMessageDialog(null, "数据库结果集无法关闭");

         }
         try {
             s.close();
         } catch (SQLException e) {

             e.printStackTrace();
             JOptionPane.showMessageDialog(null, "数据库语句无法关闭");

         }

         try {
             conn.close();
         } catch (SQLException e) {

             e.printStackTrace();
             JOptionPane.showMessageDialog(null, "与数据库的连接无法关闭");
         }
 /*
         try { // perform a clean shutdown
             DriverManager.getConnection("jdbc:derby:;shutdown=true");

         } catch (SQLException se) {

             if (((se.getErrorCode() == 50000)
                     && ("XJ015".equals(se.getSQLState())))) {
                 // we got the expected exception
                 System.out.println("Derby shut down normally");
                 // Note that for single database shutdown, the expected
                 // SQL state is "08006", and the error code is 45000.
             } else {
                 System.err.println("Derby did not shut down normally");
                 // JOptionPane.showMessageDialog(null, "数据库关闭错误");
                 se.printStackTrace();
             }
         }*/
     }

 }

程序写于大三上学期。

2016.4.12更新博客。

END