crawler_httpurlconnection_自动编码识别

核心思想：

　　 1：从响应头中读取 【命中解流准确率最高】

　　 2：如果响应头中没有，打开流从源码中读取，【取舍，如果有一般在前30行会有，前100行中寻找】

3：如果还没有，根据字节码code位置，字符识别。【前三个字符揣测】

　　 4：最终依旧没有命中采用，大陆国标编码【概率接近于0 ,gb2312】

综合效果，尚无测试到编码有问题的站点。

 /**
      * @declare:下载 自动识别编码
      * @param url
      * @return
      * @author cphmvp
      */
     public static StringBuffer downloadHtmlAutoCode(String url) {
         StringBuffer sb = new StringBuffer();
         BufferedReader bufferReader = null;
         InputStream inputStream = null;
         BufferedInputStream bufferedInputStream = null;
         int tryNum = 0;
         while (true) {
             try {
                 if (tryNum > 1) {
                     String ecodingUrl = encodParamters(url);
                     urlModel = new URL(ecodingUrl);
                 } else {
                     urlModel = new URL(url);
                 }
                 httpURLConnection = (HttpURLConnection) urlModel
                         .openConnection();
                 httpURLConnection.setConnectTimeout(connectTimeout);
                 httpURLConnection.setReadTimeout(readTimeout);
                 // httpURLConnection.setInstanceFollowRedirects(false);
                 // httpURLConnection.setFollowRedirects(true);
                 httpURLConnection
                         .setRequestProperty("User-Agent",
                                 "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)");
                 String redirectUrl = httpURLConnection.getURL().toString();
                 if (!redirectUrl.equals(url)) {
                     LOG.info(url + "重定向后为" + redirectUrl);
                 }
                 // 得到响应流
                 inputStream = httpURLConnection.getInputStream();
                 if (null == inputStream)
                     continue;
                 String charSetHeader = httpURLConnection
                         .getHeaderField("Content-Type");
                 bufferedInputStream = new BufferedInputStream(inputStream);
                 String charSet = null;
                 // 第一步先从响应头header判断
                 if (charSetHeader != null) {
                     Pattern p = Pattern.compile("charset=[\"']?(.*)['\"]?");
                     Matcher m = p.matcher(charSetHeader);
                     if (m.find()) {
                         charSet = m.group(1).trim();
                     }
                 }
                 // System.out.println(bufferedInputStream.available() > 0);
                 // System.out.println(bufferedInputStream.markSupported());
                 // 第二步 从源码中【meta http-equiv="content-type" 】判断
                 // if (null == charSet) {
                 // charSet = getEncode(bufferedInputStream);
                 // System.out.println("---->charSet: 读流识别出来的编码" + charSet);
                 // }

                 // 排除非html格式 只有一两行的状况
                 if (null == charSet
                         && charSetHeader.toLowerCase().contains("html")) {
                     // 缓冲区设置大些， read走的信息小于 这个值，就能reset 回来。
                     bufferedInputStream.mark(102400);
                     bufferReader = new BufferedReader(new InputStreamReader(
                             bufferedInputStream));
                     int lineNum = 1;
                     String inputLine;
                     // reset 在读至流的末尾是无法生效，故限制前100行找，找不到 放弃
                     while ((inputLine = bufferReader.readLine()) != null
                             && lineNum < 100) {
                         if (inputLine.toLowerCase().contains("charset")) {
                             charSet = RegexUtils.getString(inputLine,
                                     "charset=[\"']?(.*?)[\"']", 1);
                             LOG.info("自动识别出编码：" + charSet);
                             // 第一次匹配到后 ，不再往下判断，减少判断行数，及误判概率
                             break;
                         }
                         lineNum++;
                         inputLine = null;
                     }
                     // 第三步奏 穿插补录步奏
                     if (null == charSet) {
                         byte[] head = new byte[3];
                         bufferedInputStream.read(head);
                         if (head[0] == -1 && head[1] == -2)
                             charSet = "UTF-16";
                         if (head[0] == -2 && head[1] == -1)
                             charSet = "Unicode";
                         if (head[0] == -17 && head[1] == -69 && head[2] == -65)
                             charSet = "UTF-8";
                     }

                     // 通道回溯
                     bufferedInputStream.reset();
                 }

                 // 第四步奏指向默认 utf-8
                 charSet = (charSet == null ? defaultEncoding : charSet);
                 // 第五步奏按照正确编码解码响应流
                 bufferReader = new BufferedReader(new InputStreamReader(
                         bufferedInputStream, charSet));
                 String inputLine;
                 while ((inputLine = bufferReader.readLine()) != null) {
                     sb.append(inputLine + "\n");
                     inputLine = null;
                 }
                 if (bufferReader != null)
                     try {
                         bufferReader.close();
                     } catch (IOException e) {
                         LOG.error(e);
                     }
                 if (httpURLConnection != null)
                     httpURLConnection.disconnect();
                 break;
             } catch (Exception e) {
                 if (tryNum++ == 3) {
                     LOG.error("download page error [ " + urlModel + " ] ");
                     return null;
                 }
                 LOG.warn(tryNum + "次下载失败");
             }
         }
         return sb;

     }