httpclient爬取性感美图

依赖httpclient4.2,Jsop

SemeiziCrawler.java

package kidbei.learn.crawler;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
/**
 * http://sejie.wanxun.org/post/2012-09-25/40039413449
 * @author Administrator
 *
 */
public class SemeiziCrawler {
    private static final String BASEHOST = "http://sejie.wanxun.org/";
    private static DefaultHttpClient client = ConnectionManager.getHttpClient();
    static String url = "http://sejie.wanxun.org/post/2012-09-25/40039413449";
    private static String IMGPATH = "D:\\sexpicture\\色戒美眉图"+File.separator+StringUtil.getDate();
    static int STARTPAGE = 1;
    static int PAGECOUNT = 100;

    public static void main(String[] args) {
        File f = new File(IMGPATH);
        if(!f.exists()){
            f.mkdirs();
        }
        String host = BASEHOST ;
        for(int i=STARTPAGE;i<PAGECOUNT;i++){
            if(i != 1){
                host = BASEHOST+"page/"+i;
            }
            System.out.println("进入第"+i+"页");
            String pageContext = getResultByUrl(host);
//          System.out.println(pageContext);
            List<String>articleURLS = getArticleURL(pageContext);
            for(String articleURL:articleURLS){
                String articleContext = getResultByUrl(articleURL);
                List<String> ImgURLS = getImgURLS(articleContext);
                for(String ImgURL:ImgURLS){
                    savepic(ImgURL);
                }
            }
        }
//      String articleContext = getResultByUrl(url);
//      List<String> strs = getImgURLS(articleContext);
//      for(String str:strs){
//          System.out.println(str);
//      }
    }
    /**
     * 根据url获取页面
     * @param url
     * @return
     */
    public static String getResultByUrl(String url){
        System.out.println("打开网页"+url);
        HttpGet get = new HttpGet(url);
        HttpEntity entity = null;
        HttpResponse response = null;
        try {
            response = client.execute(get);
            entity = response.getEntity();
            if(entity != null){
                InputStream is = entity.getContent();
                StringWriter sw = new StringWriter();
                IOUtils.copy(is, sw, "UTF-8");
                is.close();
                sw.close();
                return sw.toString();
            }
        } catch (Exception e) {
            System.out.println("网页打开出错");
            return null;
        }finally{
            get.abort();
            try {
                EntityUtils.consume(entity);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return null;
    }
    /**
     * 找出当前页面中所有帖子的地址
     * @param pageStr  网页字符串
     * @return
     */
    public static List<String> getArticleURL(String pageContext){
        if(pageContext == null){
            return null;
        }
        List<String> articleURLS = new ArrayList<String>();
        System.out.println("寻找帖子...........");
        try {
            Document doc = Jsoup.parseBodyFragment(pageContext);
            Elements es = doc.select("div.post");
            es = es.select("div[class=post-item type-photo]");
            es = es.select("div.meta a:containsOwn(全文)");
            for(Element e:es){
                articleURLS.add(e.attr("href"));
            }
        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }
        return articleURLS;
    }
    /**
     * 获取帖子的图片地址
     * @param articleURLS
     * @return
     */
    public static List<String> getImgURLS(String articleContext){
        List<String>ImgURLS = new ArrayList<String>();
        if(articleContext == null){
            return null;
        }
        System.out.println("获取图片地址-----------");
        Document doc = Jsoup.parse(articleContext);
        Elements es = doc.select("a[target=_blank] img[src]");
         for(Iterator<Element> i=es.iterator();i.hasNext();){
                Element e = i.next();
                ImgURLS.add(e.attr("src"));
             }
        return ImgURLS;
    }
    /**
     * 保存图片
     * @param ImgURL
     */
    public static void savepic(String ImgURL){
        if(ImgURL == null){
            return ;
        }
        HttpGet get = new HttpGet(ImgURL);
        String[] strs = ImgURL.split("/");
        String fileName = strs[strs.length-1];
        String savePath = IMGPATH+File.separator+fileName;
        HttpEntity entity = null;
        try {
            HttpResponse response = client.execute(get);
            entity = response.getEntity();
            System.out.println("保存图片>>>>.>>>>>>"+fileName);
            InputStream is = entity.getContent();
            OutputStream os = new FileOutputStream(savePath);
            IOUtils.copy(is, os);
            IOUtils.closeQuietly(os);
            IOUtils.closeQuietly(is);
        } catch (Exception e) {
            e.printStackTrace();
            System.out.println("图片保存失败");
            return ;
        }
    }
}

StringUtil.java

package kidbei.learn.crawler;

import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Random;

public class StringUtil {
    public static String getRandomString(){
        StringBuffer generateRandStr = new StringBuffer();
        Random rand = new Random();
        int length = 6;
        char ch;
        for(int i=0;i<length;i++)
        {
         int randNum = Math.abs(rand.nextInt())%26+97; // 产生97到122的随机数(a-z的键位值)
            ch = ( char ) randNum;
            generateRandStr.append( ch );
        }
        return generateRandStr.toString();
    }

    public static String getSavePath(String IMGPATH,String fileName){
        SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
        String date = sdf.format(new Date()).toString();
        if(!(fileName.endsWith(".jpg"))){
            fileName = fileName + ".jpg";
        }
        String randStr = StringUtil.getRandomString();
        return IMGPATH+File.separator+date+File.separator+randStr+fileName;
    }

    public static String getDate(){
        SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
        return sdf.format(new Date()).toString();
    }
}

ConnectionManager.java

package kidbei.learn.crawler;

import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.params.HttpParams;

public class ConnectionManager {
    static final int TIMEOUT = 20000;//连接超时时间
    static final int SO_TIMEOUT = 20000;//数据传输超时
    static String UA = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1" +
            " (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1";

    public static DefaultHttpClient getHttpClient(){
        SchemeRegistry schemeRegistry = new SchemeRegistry();
        schemeRegistry.register(
                new Scheme("http",80,PlainSocketFactory.getSocketFactory()));
        schemeRegistry.register(
                new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));

        PoolingClientConnectionManager  cm = new PoolingClientConnectionManager(schemeRegistry);
        cm.setMaxTotal(500);
        cm.setDefaultMaxPerRoute(200);

        HttpParams params = new BasicHttpParams();
        params.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,TIMEOUT);
        params.setParameter(CoreConnectionPNames.SO_TIMEOUT, SO_TIMEOUT);
        params.setParameter(CoreProtocolPNames.USER_AGENT, UA);

        DefaultHttpClient client = new DefaultHttpClient(cm,params);
        return client;
    }
}

本文转自：http://www.oschina.net/code/snippet_257479_14524#23843