MongoDBcrud操作，采集部分代码

using System;

using System.Collections.Generic;

using System.ComponentModel.Design;

using System.Linq;

using System.Text;

using System.Text.RegularExpressions;

using System.Threading.Tasks;

using CDPWIB.DAL;

using CDPWIB.Data;

using CommonUtility;

using HtmlAgilityPack;

using MongoDB.Driver;

using MongoDB.Driver.Builders;

using MongoDB.Driver.Linq;

using Newtonsoft.Json;

using Newtonsoft.Json.Linq;

using WebKit;

namespace CDPWIB.WebCollection

{

    internal class QiDianCol : INovalCollect

    {

        private int Source = Convert.ToInt32(NovalSource.QiDian);

        private readonly MongoCollection<NovalTempBase> Novalcol =

            MongoConnectionFactory.GetMongoCollction<NovalTempBase>("Noval", typeof (NovalTempBase));

        public void GetNovalTypeTemp()

        {

            try

            {

                var typecol = MongoConnectionFactory.GetMongoCollction<NovalTypeTemp>("Noval", typeof (NovalTypeTemp));

                var subcol = MongoConnectionFactory.GetMongoCollction<NovalSubType>("Noval", typeof (NovalSubType));

                // 大类 http://www.qidian.com/Javascript/qidian.bookstore.js?t=20130917

                string typeshtml =

                    NetHelper.HttpGet("http://www.qidian.com/Javascript/qidian.bookstore.js?t=20130917")

                        .Replace("/", "")

                        .Replace("&nbsp", "")

                        .Replace("\r", "")

                        .Replace("\n", "")

                        .Replace("\t", "")

                        .Replace("|", "")

                        .Replace(" ", "");

                ;

                string subtypes =

                    NetHelper.HttpGet("http://script.cmfu.com/script/BookStore.js ")

                        .Replace("&nbsp", "")

                        .Replace("\r", "")

                        .Replace("\n", "")

                        .Replace("\t", "")

                        .Replace("|", "")

                        .Replace(" ", "");

                ;

                Match mtype = Regex.Match(typeshtml, "CategoryArr:(.*?)]]",

                    RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline);

                string typesstring = mtype.Groups[].Value + "]]";

                JArray typearr = (JArray) JsonConvert.DeserializeObject(typesstring);

                //JsonTextWriter

                Match msubtype = Regex.Match(subtypes, "SubCategoryArr=(.*?);",

                    RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline);

                string subtypesstring = msubtype.Groups[].Value;

                JArray subarr = (JArray) JsonConvert.DeserializeObject(subtypesstring);

                List<NovalTypeTemp> lstypes = new List<NovalTypeTemp>();

                //CategoryArr: [["全部", "-1"], ["玄幻", "21"], ["奇幻", "1"], ["武侠", "2"], ["仙侠", "22"], ["都市", "4"], ["历史", "5"], ["军事", "6"], ["游戏", "7"]

                for (int i = ; i < typearr.Count; i++)

                {

                    if (typearr[i][].ToString() != "-1")

                    {

                        NovalTypeTemp type = new NovalTypeTemp()

                        {

                            WebNum = typearr[i][].ToString().ToInt(),

                            Name = typearr[i][].ToString(),

                            Source = Source

                        };

                        lstypes.Add(type);

                    }

                }

                IMongoQuery query = Query<NovalTypeTemp>.EQ(p => p.Source, Source);

                typecol.Remove(query);

                typecol.InsertBatch(lstypes);

                List<NovalSubType> subtypels = new List<NovalSubType>();

                foreach (var NovalTypeTemp in lstypes)

                {

                    for (int i = ; i < subarr.Count; i++)

                    {

                        var obj = subarr[i];

                        if (obj[].ToString() == NovalTypeTemp.WebNum.ToString())

                        {

                            NovalSubType subtype = new NovalSubType()

                            {

                                Name = obj[].ToString(),

                                ParentWebNum = NovalTypeTemp.WebNum,

                                WebNum = obj[].ToString().ToInt(),

                                Source = Source

                            };

                            subtypels.Add(subtype);

                        }

                    }

                }

                query = Query<NovalSubType>.EQ(p => p.Source, Source);

                subcol.Remove(query);

                subcol.InsertBatch(subtypels);

            }

            catch (Exception ex)

            {

                throw;

            }

        }

        /// <summary>

        /// 根据点击数页面查小说

        /// </summary>

        public void GetNovals()

        {

            //取1到10页

            //得到月点击排行小说。

            string sourcehtml = string.Empty;

            HtmlDocument htmldocc = new HtmlDocument();

            List<NovalTempBase> qdls = new List<NovalTempBase>();

            for (int j = ; j < ; j++)

            {

                sourcehtml =

                    NetHelper.HttpGet("http://top.qidian.com/Book/TopDetail.aspx?TopType&Time=2&PageIndex=" + j);

                ;

                htmldocc.LoadHtml(sourcehtml);

                var doc = htmldocc.GetElementbyId("textlist");

                //string tablehtml = "<table>" + doc.InnerHtml + "</table>";

                //     htmldocc.LoadHtml(tablehtml);

                //一页50列

                for (int i = ; i < ; i++)

                {

                    var trdoc = doc.SelectSingleNode("tr[" + i + "]");

                    //这里的下标，从1算起

                    var tdtype = trdoc.SelectSingleNode("td[2]/a");

                    var tdbook = trdoc.SelectSingleNode("td[3]/a[1]");

                    var tdclick = trdoc.SelectSingleNode("td[4]");

                    var tdauth = trdoc.SelectSingleNode("td[5]/a");

                    Match typematch = Regex.Match(tdtype.OuterHtml, "ChannelId=(\\d*?)&SubCategoryId=(\\d*?)'");

                    Match bookmatck = Regex.Match(tdbook.OuterHtml, "Book/(\\d*?).aspx");

                    Match authmatch = Regex.Match(tdauth.OuterHtml, "id=(\\d*?)\"");

                    int authid = authmatch.Groups[].Value.ToInt();

                    int type = typematch.Groups[].Value.ToInt();

                    int subtype = typematch.Groups[].Value.ToInt();

                    int booknum = bookmatck.Groups[].Value.ToInt();

                    string bookname = tdbook.InnerText.Trim();

                    //http://image.cmfu.com/books/3127618/3127618.jpg

                    string titleimg = "http://image.cmfu.com/books/" + booknum + "/" + booknum + ".jpg";

                   bool exist= qdls.Exists(p => p.SourceWebNum == booknum);

                    if (!exist)

                    {

                        NovalTempBase qidian = new NovalTempBase()

                        {

                            AuthName = tdauth.InnerText.Trim(),

                            AuthId = authid,

                            SubType = subtype,

                            TitleImg = titleimg,

                            Title = bookname,

                            TotalClick = tdclick.InnerText.ToInt(),

                            TotalComment = ,

                            Type = type,

                            SourceWebNum = booknum,

                            Source = Source

                        };

                        qdls.Add(qidian);

                    }

                }

            }

            PublicMethod.InsertAndUpdateNovalTmp(qdls,Source);

        }

        //public void GetNovalsByType()

        //{

        //}

        /// <summary>

        /// 得到小说章节 ，个别来源，带分卷。

        /// </summary>

        public void GetNovalChapers()

        {

            //http://sight.qq.com/book/chapterpage?uin=0&g_tk=5381&callback=_Callback&pagesize=100&pageno=2&bid=16043&_r=0.6934567329008132

            var novalcol = MongoConnectionFactory.GetMongoCollction<NovalTempBase>("Noval", typeof (NovalTempBase));

            var books = novalcol.AsQueryable().Where(p=>p.Source==Source).ToList();

            foreach (var infoQidian in books)

            {

                GetSingleNovalChapers(infoQidian.SourceWebNum);

            }

        }

        public void GetSingleNovalChapers(int novalwebnum)

        {

            IMongoQuery q2 = Query<NovalVolumeTemp>.EQ(p => p.Source, Source);

            IMongoQuery q1 = Query<NovalVolumeTemp>.EQ(p => p.NovalWebNum, novalwebnum);

            IMongoQuery[] qarray = { q1, q2 };

            IMongoQuery query = Query.And(qarray);

            var chaptercol = MongoConnectionFactory.GetMongoCollction<NovalChapterTemp>("Noval", typeof(NovalChapterTemp));

            var volumecol = MongoConnectionFactory.GetMongoCollction<NovalVolumeTemp>("Noval", typeof (NovalVolumeTemp));

            List<NovalChapterTemp> lschapters = new List<NovalChapterTemp>();

            List<NovalVolumeTemp> lsvolumes = new List<NovalVolumeTemp>();

            int chapterorder = ;

            int volumeorder = ;

            HtmlDocument htmldocc = new HtmlDocument();

            //http://read.qidian.com/BookReader/3127618.aspx

            string sourcehtml = string.Empty;

            string url = "http://read.qidian.com/BookReader/" + novalwebnum + ".aspx";

            try

            {

                sourcehtml = NetHelper.HttpGet(url);

                //目录主页

                htmldocc.LoadHtml(sourcehtml);

                var doc = htmldocc.GetElementbyId("content");

                int i = ;

                var topdoc = doc.SelectSingleNode("div[" + i + "]");

                while (topdoc != null)

                {

                    var topa = topdoc.SelectSingleNode("div/a");

                    //如果是vip章节，没有这个A标签。

                    int topnum;

                    //分卷信息

                    if (topa != null)

                    {

                        string topahtml = topa.OuterHtml;

                        //href="http://www.qidian.com/BookReader/vol,107580,486625.aspx"

                        Match m = Regex.Match(topahtml, ",(\\d*?).aspx");

                        topnum = m.Groups[].Value.ToInt();

                        var topaname = topdoc.SelectSingleNode("div/b");

                        string topname = topaname.InnerText.Trim();

                        topname = topname.Replace("&nbsp", "").Split(';')[];

                        //if(topname=="作品相关")

                        NovalVolumeTemp volume = new NovalVolumeTemp()

                        {

                            Sort = volumeorder,

                            WebNum = topnum,

                            Name = topname,

                            NovalWebNum = novalwebnum,

                            Source = Source

                        };

                        lsvolumes.Add(volume);

                        volumeorder++;

                    }

                    else

                    {

                        topnum = ;

                    }

                    var contextdoc = doc.SelectSingleNode("div[" + (i + ) + "]");

                    var chaperas = contextdoc.SelectNodes("div/ul/li/a");

                    //<a itemprop='url' href="http://read.qidian.com/BookReader/107580,20901221.aspx" title='凡人修仙传字数：84  更新时间：2008-08-01 07:54:48'><span itemprop='headline'>呵呵！终于上架了！</span></a>

                    //,(\d*?).aspx

                    string chaptername = string.Empty;

                    //章节信息

                    int chapterwebnum = ;

                    for (int x = ; x < chaperas.Count; x++)

                    {

                        var chapera = chaperas[x];

                        chaptername = chapera.InnerText.Trim();

                        Match chapmatchwebnum = Regex.Match(chapera.OuterHtml, ",(\\d*?).aspx");

                        chapterwebnum = chapmatchwebnum.Groups[].Value.ToInt();

                        NovalChapterTemp chapter = new NovalChapterTemp()

                        {

                            Name = chaptername,

                            Sort = chapterorder,

                            WebNum = chapterwebnum,

                            VolumeId = topnum

                            ,

                            NovalWebNum = novalwebnum,

                            Source = Source

                        };

                        lschapters.Add(chapter);

                        chapterorder++;

                    }

                    i += ;

                    topdoc = doc.SelectSingleNode("div[" + i + "]");

                }

                volumecol.Remove(query);

                volumecol.InsertBatch(lsvolumes);

                PublicMethod.InsertChapterTempToSQL(lschapters, Source, novalwebnum);

            }

            catch (Exception ex)

            {

                return;

            }

        }

        public void GetNovalCilckComment()

        {

            var novalcol = MongoConnectionFactory.GetMongoCollction<NovalTempBase>("Noval", typeof(NovalTempBase));

            var books = novalcol.AsQueryable().Where(p => p.Source == Source).ToList();

            string sourcehtml = string.Empty;

            string url = string.Empty;

            HtmlDocument htmldocc = new HtmlDocument();

            foreach (var novalTempBase in books)

            {

                //http://www.qidian.com/Book/3106580.aspx

                 url = "http://www.qidian.com/Book/" + novalTempBase.SourceWebNum + ".aspx";

                sourcehtml = NetHelper.HttpGet(url);

                htmldocc.LoadHtml(sourcehtml);

                var cliclickdiv = htmldocc.GetElementbyId("contentdiv");

                // /div/div/div[1]/table/tbody/tr/td[1]

                var clickcount =

                    cliclickdiv.SelectSingleNode("div/div[1]/table/tr/td[1]")

                        .InnerText.Replace("总点击", "")

                        .Replace("：", "").Trim();

                int click = Convert.ToInt32(clickcount);

            //    string urlcom = "http://forum.qidian.com/NewForum/List.aspx?BookId=3106580";

            ////http://forum.qidian.com/NewForum/List.aspx?BookId=3106580

            // //   http://c.pingba.qidian.com/BookComment.aspx?BookId=3106580

            //    url = "http://c.pingba.qidian.com/BookComment.aspx?" + novalTempBase.SourceWebNum;

            //    sourcehtml = NetHelper.HttpGet(url);

            //    htmldocc.LoadHtml(sourcehtml);

                novalTempBase.TotalClick = click;

                novalcol.Save(novalTempBase);

            }

                //目录主页

        }

    }

}

MongoDBcrud操作，采集部分代码的更多相关文章

SELECT控件操作的JS代码示例
SELECT控件操作的JS代码示例 1 检测是否有选中 if(objSelect.selectedIndex > -1) { //说明选中 } else { //说明没有选中 } 2.动态创建s ...
PHP禁止同一IP频繁访问以防止网站被防攻击或采集的代码
PHP禁止同一IP频繁访问以防止网站被防攻击或采集的代码 <?php /* *通过禁止IP频繁访问防止网站被防攻击代码*design by www.scutephp.com*/header('C ...
C#开发中使用Npoi操作excel实例代码
C#开发中使用Npoi操作excel实例代码出处:西西整理作者:西西日期:2012/11/16 9:35:50 [大中小] 评论: 0 | 我要发表看法 Npoi 是什么? 1.整个Exce ...
30个php操作redis常用方法代码例子
From: http://www.jb51.net/article/51884.htm 这篇文章主要介绍了30个php操作redis常用方法代码例子,本文其实不止30个方法,可以操作string类型. ...
php foreach 操作数组的代码
php foreach 操作数组的代码. foreach()有两种用法: foreach(array_name as $value) { statement; } 这里的array_na ...
30 个 php 操作 redis 常用方法代码例子
这篇文章主要介绍了 30 个 php 操作 redis 常用方法代码例子 , 本文其实不止 30 个方法 , 可以操作 string 类型. list 类型和 set 类型的数据 , 需要的朋友可以参 ...
Redis：安装、配置、操作和简单代码实例(C语言Client端)
Redis:安装.配置.操作和简单代码实例(C语言Client端) - hj19870806的专栏 - 博客频道 - CSDN.NET Redis:安装.配置.操作和简单代码实例(C语言Client端 ...
Installshield停止操作系统进程的代码--IS5版本适用
原文:Installshield停止操作系统进程的代码--IS5版本适用出处:http://www.installsite.org/pages/en/isp_ext.htm这个地址上有不少好东西,有 ...
Installshield停止操作系统进程的代码 --IS6及以上版本适用
原文:Installshield停止操作系统进程的代码 --IS6及以上版本适用 setup.rul的代码 Code;end;///////////////////////////////////// ...
C# FTP操作类的代码
如下代码是关于C# FTP操作类的代码.using System;using System.Collections.Generic;using System.Text;using System.Net ...

随机推荐

17.3.10--->关于数值溢出问题
取值范围: short.int.long 占用的字节数不同,所能表示的数值范围也不同.以32位平台为例,下面是它们的取值范围: 数据类型所占字 ...
gradle问题
1, my gradle version is 4.6 . in project.gradle : change dependencies { classpath 'com.android.tools ...
第二季第八天 part2
for (let i = 0; i < 3; i++) { log(i) } log(i) // 结果是 undefined let和const的作用域只在花括号内 let和const不能重复声 ...
NOIpDairy
Day 0 水水比赛 Day 1 写写Dp Part1:Dp基础练习 [HNOI2002]公交车路线秒切,点数这么少,N这么大,目测O(N)+暴力更新 5min写完 P3842 [TJOI2007] ...
Python程序中的进程操作--—--开启多进程
Python程序中的进程操作-----开启多进程之前我们已经了解了很多进程相关的理论知识,了解进程是什么应该不再困难了,刚刚我们已经了解了,运行中的程序就是一个进程.所有的进程都是通过它的父进程来创 ...
4. 监控利器nagios手把手企业级实战第三部
1.nagios图形监控显示和管理服务器虽然能显示,能报警.但是我们企业工作中需要一个历史趋势图. nagios只开放核心,插件是单独的形式,图像也一样,是插件或者整合的方式.所以可能看起来很多,这 ...
3）PHP基本语法和变量基础，区分大小写
(1)语法环境: PHP嵌入到html到代码中: . <?php 这里是PHP代码 ?> 标准形式: <script language='php'> 这里是PHP代码 < ...
PyTorch基础——预测共享单车的使用量
预处理实验数据读取数据下载数据网盘链接:https://pan.baidu.com/s/1n_FtZjAswWR9rfuI6GtDhA 提取码:y4fb #导入需要使用的库 import num ...
tap点击一次，内部程序执行两次，多次
调试过程发现,使用 $(document).on('tap', '.children2', function () { //内部程序 }) 点击children2的时候,程序在里面执行了两次.百度得到 ...
D - Project Presentation(DFS序+倍增LCA）
You are given a tree that represents a hierarchy in a company, where the parent of node u is their d ...

MongoDBcrud操作，采集部分代码

MongoDBcrud操作，采集部分代码的更多相关文章

随机推荐

热门专题