java爬虫简单实现

package WebSpider;

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStreamReader;

import java.net.MalformedURLException;

import java.net.URL;

import java.nio.charset.Charset;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

/**

• 网络爬虫
• @author Vcanccc

/

public class WebSpiderTest {

/*

* 获得urlStr对应网络内容

* @param urlStr

* @return

*/

public static String getURLContent(String urlStr, String charset){

StringBuilder sb = new StringBuilder();

try {

URL url = new URL(urlStr);

BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream(),Charset.forName(charset)));

String temp = "";

while((temp = reader.readLine()) != null)

{

sb.append(temp);

}

} catch (MalformedURLException e) {

e.printStackTrace();

} catch (IOException e) {

e.printStackTrace();

}

return sb.toString();

}

public static void main(String[] args) {
    String destStr = getURLContent("https://www.taobao.com","utf-8");

//    Pattern p = Pattern.compile("<a[\\s\\S]+?</a>");
    Pattern p = Pattern.compile("href=\"(.+?)\"");
    Matcher m = p.matcher(destStr);

    while(m.find()){
        System.out.println(m.group(1));
    }
}

}