Java解析采集模块

 package step3;

 import java.io.BufferedReader;

 import java.io.BufferedWriter;

 import java.io.File;

 import java.io.FileReader;

 import java.io.FileWriter;

 import java.io.IOException;

 import java.io.InputStream;

 import java.io.InputStreamReader;

 import java.io.PrintWriter;

 import java.sql.ResultSet;

 import java.sql.SQLException;

 import java.sql.Statement;

 import java.util.ArrayList;

 import java.util.Calendar;

 import java.util.List;

 import org.apache.commons.httpclient.HttpClient;

 import org.apache.commons.httpclient.methods.GetMethod;

 import org.apache.commons.httpclient.methods.PostMethod;

 import org.json.JSONObject;

 import org.jsoup.Jsoup;

 import org.jsoup.nodes.Document;

 import org.jsoup.select.Elements;

 import bean.Porn;

 import util.DBConnection;

 /**

  *

  * @ClassName: quhao

  * @Description: 91porn地址解析

  * @author zeze

  * @date 2016年06月30日 下午7:55:31

  *

  */

 public class porn91 {

     private static String cookie = "incap_ses_401_649914=31EbXVOgx0r6Ql5TmqOQBdjxdFcAAAAAu7MrrqICFZvpjsIw5VriGQ==; incap_ses_434_649914=wx2HcnWH7GDQCChRweAFBt/xdFcAAAAAczn9Ohl2VBPqxEd8kRi2GA==; incap_ses_407_649914=U4VYNM5iO1l1H0VP7/SlBWXydFcAAAAAifL73Yq/OnIgRqKWiWPqUg==; incap_ses_406_649914=8Ub/DfvqEGs9L9gFemeiBWEKdVcAAAAA+aBeDqKyWw37Sv+KZ4cdlA==; incap_ses_432_649914=bLzAYBXvVG0kSU6wyMX+BWUKdVcAAAAAZW+uykXgylzu/dZOu7IDWw==; _ga=GA1.2.1738858661.1466764840; _gat=1; visid_incap_649914=2hb3ym0OQ9C7sr1krqKCQTUObVcAAAAAQUIPAAAAAADQQCM/QP5jhCXO3+mlIKmg; incap_ses_199_649914=RkWbbfybyCoL2fxKs/3CAqIbdVcAAAAAOa+RJFdt35NV8xtM8MbP8Q==; session=eyJfZnJlc2giOmZhbHNlLCJjc3JmX3Rva2VuIjp7IiBiIjoiTkdFek9HRmtNakkxTldVM05EVXpZMkZoTldKaE5tWXpOV014TlRBNU1UZ3dPVGcyTkRNMU5BPT0ifX0.ClatMQ.INJmWYMZ8T220CgsSTcfpHhTxXI";

     private static String cookie2 = "incap_ses_401_649914=31EbXVOgx0r6Ql5TmqOQBdjxdFcAAAAAu7MrrqICFZvpjsIw5VriGQ==; incap_ses_434_649914=wx2HcnWH7GDQCChRweAFBt/xdFcAAAAAczn9Ohl2VBPqxEd8kRi2GA==; incap_ses_407_649914=U4VYNM5iO1l1H0VP7/SlBWXydFcAAAAAifL73Yq/OnIgRqKWiWPqUg==; incap_ses_406_649914=8Ub/DfvqEGs9L9gFemeiBWEKdVcAAAAA+aBeDqKyWw37Sv+KZ4cdlA==; incap_ses_432_649914=bLzAYBXvVG0kSU6wyMX+BWUKdVcAAAAAZW+uykXgylzu/dZOu7IDWw==; _ga=GA1.2.1738858661.1466764840; _gat=1; visid_incap_649914=2hb3ym0OQ9C7sr1krqKCQTUObVcAAAAAQUIPAAAAAADQQCM/QP5jhCXO3+mlIKmg; incap_ses_199_649914=RkWbbfybyCoL2fxKs/3CAqIbdVcAAAAAOa+RJFdt35NV8xtM8MbP8Q==; session=eyJfZnJlc2giOmZhbHNlLCJjc3JmX3Rva2VuIjp7IiBiIjoiTkdFek9HRmtNakkxTldVM05EVXpZMkZoTldKaE5tWXpOV014TlRBNU1UZ3dPVGcyTkRNMU5BPT0ifX0.ClatMw.6MGC1jX7mgjsChpGFBd-xHTv9ZU";

     private static String Token = "1467296187##60ecf40d9328862cc6cd6a478adfc72ee0554050";

     private static String Url = "http://freeget.co/video/extraction";

     private static String url001 = null;

     private static String dirfile = "F:/91porn/91url.csv";

     private static String destfile = "F:/91porn/data.txt";

     private static int cnt0 = 0;

     private static String num = null;

     private static String title = null;

     private static String time = null;

     private static String longtime = null;

     private static String viewnum = null;

     private static String Parurl = null;// "http://www.91porn.com/view_video.php?viewkey=c5ec60d0da8c8fbdb180&page=4&viewtype=basic&category=mr";

     public static void main(String[] args) throws InterruptedException {

         File file = new File(dirfile);

         FileReader reader = null;

         BufferedReader br = null;

         try {

             reader = new FileReader(file);

             br = new BufferedReader(reader);

             String str = null;

             String[] strArr = null;

             int cnt = 0;

             while ((str = br.readLine()) != null) {

                 // System.out.println(str);

                 strArr = str.split(",");

                 if (strArr.length != 7)

                     continue;

                 num = strArr[0];

                 title = strArr[1];

                 time = strArr[2];

                 longtime = strArr[4];

                 viewnum = strArr[5];

                 Parurl = strArr[6];

                 cnt++;

                 System.out.println(num + "," + title + "," + time);

                 func_step1();

             }

             System.out.println("采集结束,总共：" + cnt + "条,成功写入" + cnt0 + "条");

         } catch (Exception e) {

             // TODO: handle exception

             e.printStackTrace();

         } finally {

             if (br != null) {

                 try {

                     br.close();

                 } catch (Exception e2) {

                     // TODO: handle exception

                     e2.printStackTrace();

                 }

             }

             if (reader != null) {

                 try {

                     reader.close();

                 } catch (Exception e2) {

                     // TODO: handle exception

                     e2.printStackTrace();

                 }

             }

         }

     }

     private static void func_step1() {

         HttpClient httpClient = new HttpClient();

         try {

             PostMethod postMethod = new PostMethod(Url);

             postMethod.getParams().setContentCharset("utf-8");

             // 每次访问需授权的网址时需 cookie 作为通行证

             postMethod.setRequestHeader("cookie", cookie);

             postMethod.setRequestHeader("X-CSRFToken", Token);

             postMethod.setRequestHeader("Accept-Language", "zh-CN,zh;q=0.8");

             postMethod.setRequestHeader("Host", "freeget.co");

             postMethod.setRequestHeader("Referer", "http://freeget.co/");

             postMethod.setRequestHeader("User-Agent",

                     "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0) QQBrowser/9.2.5063.400");

             postMethod.setParameter("url", Parurl);

             int statusCode = httpClient.executeMethod(postMethod);// 返回状态码200为成功，500为服务器端发生运行错误

             System.out.println("返回状态码：" + statusCode);

             // 打印出返回数据，检验一下是否成功

             String result = postMethod.getResponseBodyAsString();

             if (statusCode == 200) {

                 // 解析成功，取得token和view_key

                 JSONObject a = new JSONObject(result);

                 url001 = "http://freeget.co/video/" + a.get("view_key") + "/" + a.get("token");

                 System.out.println("视频解析地址：" + url001);

                 func_step2(url001);

             }

         } catch (Exception e) {

             e.printStackTrace();

         }

     }

     private static void func_step2(String url) {

         HttpClient httpClient = new HttpClient();

         try {

             GetMethod getMethod = new GetMethod(url);

             getMethod.getParams().setContentCharset("utf-8");

             getMethod.setRequestHeader("cookie", cookie2);

             getMethod.setRequestHeader("Accept-Language", "zh-cn");

             getMethod.setRequestHeader("User-Agent",

                     "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0) QQBrowser/9.2.5063.400");

             int statusCode = httpClient.executeMethod(getMethod);// 返回状态码200为成功，500为服务器端发生运行错误

             // System.out.println("返回状态码：" + statusCode);

             // 打印出返回数据，检验一下是否成功

             InputStream inputStream = getMethod.getResponseBodyAsStream();

             BufferedReader br = new BufferedReader(new InputStreamReader(inputStream));

             StringBuffer stringBuffer = new StringBuffer();

             String str = "";

             while ((str = br.readLine()) != null) {

                 stringBuffer.append(str);

             }

             if (statusCode == 200) {

                 Document doc = Jsoup.parse(stringBuffer.toString());

                 Elements name = doc.select("a");

                 String playurl = name.get(4).text();

                 System.out.println("在线播放地址：" + playurl);

                 writefile(playurl);

                 cnt0++;

             }

         } catch (Exception e) {

             e.printStackTrace();

         }

     }

     private static void writefile(String url) {

         FileWriter fw = null;

         BufferedWriter bw = null;

         PrintWriter pw = null;

         try {

             fw = new FileWriter(new File(destfile), true);

             bw = new BufferedWriter(fw);

             pw = new PrintWriter(bw);

             pw.write(num + ',' + title + ',' + time + ',' + longtime + ',' + viewnum + ',' + url + "\r\n");

         } catch (IOException e) {

             // TODO Auto-generated catch block

             e.printStackTrace();

         } finally {

             if (pw != null) {

                 pw.close();

             }

             if (bw != null) {

                 try {

                     bw.close();

                 } catch (IOException e) {

                     // TODO Auto-generated catch block

                     e.printStackTrace();

                 }

             }

             if (fw != null) {

                 try {

                     fw.close();

                 } catch (IOException e) {

                     // TODO Auto-generated catch block

                     e.printStackTrace();

                 }

             }

         }

     }

     public List<Porn> QueryAllBook() {

         java.sql.Connection connection = DBConnection.getConnection();

         String sql = "select * from porn where status=0";

         java.sql.PreparedStatement pstmt = DBConnection.getPreparedStatement(connection, sql);

         List<Porn> pornlist = new ArrayList<Porn>();

         System.out.println(sql);

         try {

             Statement stmt = connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_READ_ONLY);

             java.sql.ResultSet rs = stmt.executeQuery(sql);

             while (rs.next()) {

                 Porn porn = new Porn();

                 porn.setNum(rs.getString(1));

                 porn.setTitle(rs.getString(2));

                 porn.setTime(rs.getString(3));

                 porn.setViewkey(rs.getString(4));

                 porn.setLongtime(rs.getString(5));

                 porn.setViewnum(rs.getString(6));

                 porn.setParurl(rs.getString(7));

                 pornlist.add(porn);

             }

             rs.last();

         } catch (SQLException e) {

             e.printStackTrace();

         } finally {

             DBConnection.close(connection, pstmt, null);

         }

         return pornlist;

     }

 }

Java解析采集模块的更多相关文章

java解析xml的三种方法
java解析XML的三种方法 1.SAX事件解析 package com.wzh.sax; import org.xml.sax.Attributes; import org.xml.sax.SAXE ...
atitit.java解析sql语言解析器解释器的实现
atitit.java解析sql语言解析器解释器的实现 1. 解析sql的本质:实现一个4gl dsl编程语言的编译器 1 2. 解析sql的主要的流程,词法分析,而后进行语法分析,语义分析,构建sq ...
java 解析XML文档
Java 解析XML文档一.解析XML文档方式: 1.DOM方式:将整个XML文档读取到内存中,按照XML文件的树状结构图进行解析. 2.SAX方式:基于事件的解析,只需要加载XML中的部分数据,优 ...
Java 解析 XML
Java 解析 XML 标签: Java基础 XML解析技术有两种 DOM SAX DOM方式根据XML的层级结构在内存中分配一个树形结构,把XML的标签,属性和文本等元素都封装成树的节点对象优点 ...
JAVA解析XML的四种方式
java解析xml文件四种方式 1.介绍 1)DOM(JAXP Crimson解析器) DOM是用与平台和语言无关的方式表示XML文档的官方W3C标准.DOM是以层次结构组织的节点或信息片断的集合.这 ...
JSON 之JAVA 解析
一. JSON (JavaScript Object Notation)一种简单的数据格式,比xml更轻巧. Json建构于两种结构: 1.“名称/值”对的集合(A collection ...
Java解析XML汇总（DOM/SAX/JDOM/DOM4j/XPath）
[目录] 一.[基础知识——扫盲] 二.[DOM.SAX.JDOM.DOM4j简单使用介绍] 三.[性能测试] 四.[对比] 五.[小插曲XPath] 六.[补充] 关键字:Java解析xml.解析x ...
Java 解析epub格式电子书，helloWorld程序，附带源程序和相关jar包
秀才坤坤出品一.epub格式电子书相关材料和源码均在链接中可以下载:http://pan.baidu.com/s/1bnm8YXT 包括 1.JAVA项目工程test_epub,里面包括了jar包 ...
Java解析XML文档(简单实例)——dom解析xml
一.前言用Java解析XML文档,最常用的有两种方法:使用基于事件的XML简单API(Simple API for XML)称为SAX和基于树和节点的文档对象模型(Document Object ...

随机推荐

Java数据结构——用链表实现栈
//================================================= // File Name : LinkStack_demo //---------------- ...
JavaScript学习笔记——对象知识点
javascript对象的遍历.内存分布和封装特性一.javascript对象遍历 1.javascript属性访问对象.属性对象[属性] //字符串格式 //javascript属性的访问方法 ...
斗鱼的sidebar的实现简陋的demo
斗鱼的sidebar的实现简陋的demo <!DOCTYPE html> <html> <head lang="en"> <meta ch ...
Lua之元表
Lua之metatable 一.元表 Lua中的每个值都有一套预定义的操作集合,也可以通过metatable(元表)来定义一个值的行为,metatable包含了一组meatmethod(元方法). L ...
Java Runtime.availableProcessors()方法
Java Runtime.availableProcessors()方法用法实例教程. 描述 java.lang.Runtime.availableProcessors() 方法返回到Java虚拟 ...
linux 关机命令总结
linux下常用的关机命令有:shutdown.halt.poweroff.init:重启命令有:reboot.下面本文就主要介绍一些常用的关机命令以及各种关机命令之间的区别和具体用法. 首先来看一下 ...
Java学习笔记（二）不定时更新
Java语言画图 package cn.witksy.dev; import javax.imageio.ImageIO; import java.awt.*; import java.awt.ima ...
Node.SelectNodes
http://www.crifan.com/csharp_under_some_node_search_specific_child_node/ https://msdn.microsoft.com/ ...
Python中MySQLdb的事务处理
今天写了个tool,目的是把csv中的数据插入到数据库中去.其中有一部分,是需要分别向两张表中插入两条数据,如果第二张表中的数据已经存在,那么第一张表中的数据也不需要插入. 然后通过百度查找发现,其实 ...
SQL如何将A,B,C替换为'A','B','C'
因为涉及到逗号,和单引号' 本来想一次转换成功, 但是最后貌似没有好的办法, 只有分两次完成了 select REPLACE(REPLACE('A,B,C',',','>,>'),'> ...

Java解析采集模块

Java解析采集模块的更多相关文章

随机推荐

热门专题