Java解析采集模块

 package step3;

 import java.io.BufferedReader;

 import java.io.BufferedWriter;

 import java.io.File;

 import java.io.FileReader;

 import java.io.FileWriter;

 import java.io.IOException;

 import java.io.InputStream;

 import java.io.InputStreamReader;

 import java.io.PrintWriter;

 import java.sql.ResultSet;

 import java.sql.SQLException;

 import java.sql.Statement;

 import java.util.ArrayList;

 import java.util.Calendar;

 import java.util.List;

 import org.apache.commons.httpclient.HttpClient;

 import org.apache.commons.httpclient.methods.GetMethod;

 import org.apache.commons.httpclient.methods.PostMethod;

 import org.json.JSONObject;

 import org.jsoup.Jsoup;

 import org.jsoup.nodes.Document;

 import org.jsoup.select.Elements;

 import bean.Porn;

 import util.DBConnection;

 /**

  *

  * @ClassName: quhao

  * @Description: 91porn地址解析

  * @author zeze

  * @date 2016年06月30日 下午7:55:31

  *

  */

 public class porn91 {

     private static String cookie = "incap_ses_401_649914=31EbXVOgx0r6Ql5TmqOQBdjxdFcAAAAAu7MrrqICFZvpjsIw5VriGQ==; incap_ses_434_649914=wx2HcnWH7GDQCChRweAFBt/xdFcAAAAAczn9Ohl2VBPqxEd8kRi2GA==; incap_ses_407_649914=U4VYNM5iO1l1H0VP7/SlBWXydFcAAAAAifL73Yq/OnIgRqKWiWPqUg==; incap_ses_406_649914=8Ub/DfvqEGs9L9gFemeiBWEKdVcAAAAA+aBeDqKyWw37Sv+KZ4cdlA==; incap_ses_432_649914=bLzAYBXvVG0kSU6wyMX+BWUKdVcAAAAAZW+uykXgylzu/dZOu7IDWw==; _ga=GA1.2.1738858661.1466764840; _gat=1; visid_incap_649914=2hb3ym0OQ9C7sr1krqKCQTUObVcAAAAAQUIPAAAAAADQQCM/QP5jhCXO3+mlIKmg; incap_ses_199_649914=RkWbbfybyCoL2fxKs/3CAqIbdVcAAAAAOa+RJFdt35NV8xtM8MbP8Q==; session=eyJfZnJlc2giOmZhbHNlLCJjc3JmX3Rva2VuIjp7IiBiIjoiTkdFek9HRmtNakkxTldVM05EVXpZMkZoTldKaE5tWXpOV014TlRBNU1UZ3dPVGcyTkRNMU5BPT0ifX0.ClatMQ.INJmWYMZ8T220CgsSTcfpHhTxXI";

     private static String cookie2 = "incap_ses_401_649914=31EbXVOgx0r6Ql5TmqOQBdjxdFcAAAAAu7MrrqICFZvpjsIw5VriGQ==; incap_ses_434_649914=wx2HcnWH7GDQCChRweAFBt/xdFcAAAAAczn9Ohl2VBPqxEd8kRi2GA==; incap_ses_407_649914=U4VYNM5iO1l1H0VP7/SlBWXydFcAAAAAifL73Yq/OnIgRqKWiWPqUg==; incap_ses_406_649914=8Ub/DfvqEGs9L9gFemeiBWEKdVcAAAAA+aBeDqKyWw37Sv+KZ4cdlA==; incap_ses_432_649914=bLzAYBXvVG0kSU6wyMX+BWUKdVcAAAAAZW+uykXgylzu/dZOu7IDWw==; _ga=GA1.2.1738858661.1466764840; _gat=1; visid_incap_649914=2hb3ym0OQ9C7sr1krqKCQTUObVcAAAAAQUIPAAAAAADQQCM/QP5jhCXO3+mlIKmg; incap_ses_199_649914=RkWbbfybyCoL2fxKs/3CAqIbdVcAAAAAOa+RJFdt35NV8xtM8MbP8Q==; session=eyJfZnJlc2giOmZhbHNlLCJjc3JmX3Rva2VuIjp7IiBiIjoiTkdFek9HRmtNakkxTldVM05EVXpZMkZoTldKaE5tWXpOV014TlRBNU1UZ3dPVGcyTkRNMU5BPT0ifX0.ClatMw.6MGC1jX7mgjsChpGFBd-xHTv9ZU";

     private static String Token = "1467296187##60ecf40d9328862cc6cd6a478adfc72ee0554050";

     private static String Url = "http://freeget.co/video/extraction";

     private static String url001 = null;

     private static String dirfile = "F:/91porn/91url.csv";

     private static String destfile = "F:/91porn/data.txt";

     private static int cnt0 = 0;

     private static String num = null;

     private static String title = null;

     private static String time = null;

     private static String longtime = null;

     private static String viewnum = null;

     private static String Parurl = null;// "http://www.91porn.com/view_video.php?viewkey=c5ec60d0da8c8fbdb180&page=4&viewtype=basic&category=mr";

     public static void main(String[] args) throws InterruptedException {

         File file = new File(dirfile);

         FileReader reader = null;

         BufferedReader br = null;

         try {

             reader = new FileReader(file);

             br = new BufferedReader(reader);

             String str = null;

             String[] strArr = null;

             int cnt = 0;

             while ((str = br.readLine()) != null) {

                 // System.out.println(str);

                 strArr = str.split(",");

                 if (strArr.length != 7)

                     continue;

                 num = strArr[0];

                 title = strArr[1];

                 time = strArr[2];

                 longtime = strArr[4];

                 viewnum = strArr[5];

                 Parurl = strArr[6];

                 cnt++;

                 System.out.println(num + "," + title + "," + time);

                 func_step1();

             }

             System.out.println("采集结束,总共：" + cnt + "条,成功写入" + cnt0 + "条");

         } catch (Exception e) {

             // TODO: handle exception

             e.printStackTrace();

         } finally {

             if (br != null) {

                 try {

                     br.close();

                 } catch (Exception e2) {

                     // TODO: handle exception

                     e2.printStackTrace();

                 }

             }

             if (reader != null) {

                 try {

                     reader.close();

                 } catch (Exception e2) {

                     // TODO: handle exception

                     e2.printStackTrace();

                 }

             }

         }

     }

     private static void func_step1() {

         HttpClient httpClient = new HttpClient();

         try {

             PostMethod postMethod = new PostMethod(Url);

             postMethod.getParams().setContentCharset("utf-8");

             // 每次访问需授权的网址时需 cookie 作为通行证

             postMethod.setRequestHeader("cookie", cookie);

             postMethod.setRequestHeader("X-CSRFToken", Token);

             postMethod.setRequestHeader("Accept-Language", "zh-CN,zh;q=0.8");

             postMethod.setRequestHeader("Host", "freeget.co");

             postMethod.setRequestHeader("Referer", "http://freeget.co/");

             postMethod.setRequestHeader("User-Agent",

                     "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0) QQBrowser/9.2.5063.400");

             postMethod.setParameter("url", Parurl);

             int statusCode = httpClient.executeMethod(postMethod);// 返回状态码200为成功，500为服务器端发生运行错误

             System.out.println("返回状态码：" + statusCode);

             // 打印出返回数据，检验一下是否成功

             String result = postMethod.getResponseBodyAsString();

             if (statusCode == 200) {

                 // 解析成功，取得token和view_key

                 JSONObject a = new JSONObject(result);

                 url001 = "http://freeget.co/video/" + a.get("view_key") + "/" + a.get("token");

                 System.out.println("视频解析地址：" + url001);

                 func_step2(url001);

             }

         } catch (Exception e) {

             e.printStackTrace();

         }

     }

     private static void func_step2(String url) {

         HttpClient httpClient = new HttpClient();

         try {

             GetMethod getMethod = new GetMethod(url);

             getMethod.getParams().setContentCharset("utf-8");

             getMethod.setRequestHeader("cookie", cookie2);

             getMethod.setRequestHeader("Accept-Language", "zh-cn");

             getMethod.setRequestHeader("User-Agent",

                     "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0) QQBrowser/9.2.5063.400");

             int statusCode = httpClient.executeMethod(getMethod);// 返回状态码200为成功，500为服务器端发生运行错误

             // System.out.println("返回状态码：" + statusCode);

             // 打印出返回数据，检验一下是否成功

             InputStream inputStream = getMethod.getResponseBodyAsStream();

             BufferedReader br = new BufferedReader(new InputStreamReader(inputStream));

             StringBuffer stringBuffer = new StringBuffer();

             String str = "";

             while ((str = br.readLine()) != null) {

                 stringBuffer.append(str);

             }

             if (statusCode == 200) {

                 Document doc = Jsoup.parse(stringBuffer.toString());

                 Elements name = doc.select("a");

                 String playurl = name.get(4).text();

                 System.out.println("在线播放地址：" + playurl);

                 writefile(playurl);

                 cnt0++;

             }

         } catch (Exception e) {

             e.printStackTrace();

         }

     }

     private static void writefile(String url) {

         FileWriter fw = null;

         BufferedWriter bw = null;

         PrintWriter pw = null;

         try {

             fw = new FileWriter(new File(destfile), true);

             bw = new BufferedWriter(fw);

             pw = new PrintWriter(bw);

             pw.write(num + ',' + title + ',' + time + ',' + longtime + ',' + viewnum + ',' + url + "\r\n");

         } catch (IOException e) {

             // TODO Auto-generated catch block

             e.printStackTrace();

         } finally {

             if (pw != null) {

                 pw.close();

             }

             if (bw != null) {

                 try {

                     bw.close();

                 } catch (IOException e) {

                     // TODO Auto-generated catch block

                     e.printStackTrace();

                 }

             }

             if (fw != null) {

                 try {

                     fw.close();

                 } catch (IOException e) {

                     // TODO Auto-generated catch block

                     e.printStackTrace();

                 }

             }

         }

     }

     public List<Porn> QueryAllBook() {

         java.sql.Connection connection = DBConnection.getConnection();

         String sql = "select * from porn where status=0";

         java.sql.PreparedStatement pstmt = DBConnection.getPreparedStatement(connection, sql);

         List<Porn> pornlist = new ArrayList<Porn>();

         System.out.println(sql);

         try {

             Statement stmt = connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_READ_ONLY);

             java.sql.ResultSet rs = stmt.executeQuery(sql);

             while (rs.next()) {

                 Porn porn = new Porn();

                 porn.setNum(rs.getString(1));

                 porn.setTitle(rs.getString(2));

                 porn.setTime(rs.getString(3));

                 porn.setViewkey(rs.getString(4));

                 porn.setLongtime(rs.getString(5));

                 porn.setViewnum(rs.getString(6));

                 porn.setParurl(rs.getString(7));

                 pornlist.add(porn);

             }

             rs.last();

         } catch (SQLException e) {

             e.printStackTrace();

         } finally {

             DBConnection.close(connection, pstmt, null);

         }

         return pornlist;

     }

 }

Java解析采集模块的更多相关文章

java解析xml的三种方法
java解析XML的三种方法 1.SAX事件解析 package com.wzh.sax; import org.xml.sax.Attributes; import org.xml.sax.SAXE ...
atitit.java解析sql语言解析器解释器的实现
atitit.java解析sql语言解析器解释器的实现 1. 解析sql的本质:实现一个4gl dsl编程语言的编译器 1 2. 解析sql的主要的流程,词法分析,而后进行语法分析,语义分析,构建sq ...
java 解析XML文档
Java 解析XML文档一.解析XML文档方式: 1.DOM方式:将整个XML文档读取到内存中,按照XML文件的树状结构图进行解析. 2.SAX方式:基于事件的解析,只需要加载XML中的部分数据,优 ...
Java 解析 XML
Java 解析 XML 标签: Java基础 XML解析技术有两种 DOM SAX DOM方式根据XML的层级结构在内存中分配一个树形结构,把XML的标签,属性和文本等元素都封装成树的节点对象优点 ...
JAVA解析XML的四种方式
java解析xml文件四种方式 1.介绍 1)DOM(JAXP Crimson解析器) DOM是用与平台和语言无关的方式表示XML文档的官方W3C标准.DOM是以层次结构组织的节点或信息片断的集合.这 ...
JSON 之JAVA 解析
一. JSON (JavaScript Object Notation)一种简单的数据格式,比xml更轻巧. Json建构于两种结构: 1.“名称/值”对的集合(A collection ...
Java解析XML汇总（DOM/SAX/JDOM/DOM4j/XPath）
[目录] 一.[基础知识——扫盲] 二.[DOM.SAX.JDOM.DOM4j简单使用介绍] 三.[性能测试] 四.[对比] 五.[小插曲XPath] 六.[补充] 关键字:Java解析xml.解析x ...
Java 解析epub格式电子书，helloWorld程序，附带源程序和相关jar包
秀才坤坤出品一.epub格式电子书相关材料和源码均在链接中可以下载:http://pan.baidu.com/s/1bnm8YXT 包括 1.JAVA项目工程test_epub,里面包括了jar包 ...
Java解析XML文档(简单实例)——dom解析xml
一.前言用Java解析XML文档,最常用的有两种方法:使用基于事件的XML简单API(Simple API for XML)称为SAX和基于树和节点的文档对象模型(Document Object ...

随机推荐

关于Linux发行版的选择
Linux发行版很多,分为以RedHat为代表的商业发行版和以Debian为代表的免费发行版.前者典型版本有CentOS.Fedora.SUSE等,后者的典型版本有Ubuntu等 CentOS.Ubu ...
SRS(simple-rtmp-server)
1.Push h.264 to rtmp server from tcp.
Asp.NET的Trace追踪
http://m.blog.csdn.net/article/details?id=7026402 当我们扑捉程序错误时,调试器是开发者们最得力的助手.然而,ASP.NET的跟踪,在调试时是一个很棒的 ...
Using Friendly URLs in ASP.NET Web Forms
Introduction Websites often need to generate SEO friendly URLs. In ASP.NET Web Forms applications, a ...
owin要跑起来
必须安装 Microsoft.Owin.Host.SystemWeb
MongoVUE
MongoVUE运行界面如下:
Linux下删除命令
linux删除目录很简单使用rm -rf命令即可.直接rm就可以了,不过要加两个参数-rf 即:rm -rf 目录名字(警告:不作任何提示,且不可恢复,不要说没提醒哦!严重警告!!!)-r 就是向下 ...
Yii2 redis与cache
原文地址:http://www.myexception.cn/php/1974979.html composer require yiisoft/yii2-redis 安装后使用超简单,打开 comm ...
yii2 登录、退出、自动登录
自动登录的原理很简单.主要就是利用cookie来实现的在第一次登录的时候,如果登录成功并且选中了下次自动登录,那么就会把用户的认证信息保存到cookie中,cookie的有效期为1年或者几个月. 在下 ...
[Angularjs]ng-include 包含
写在前面有时我们需要动态的创建一些标签,或者在收到服务端返回的json,创建一些标签然后找到页面上的元素,通过innerHTML写入到页面上面.angularjs也为我们提供了一种比较方便操作方式, ...

Java解析采集模块

Java解析采集模块的更多相关文章

随机推荐

热门专题