C#获取网页信息并存入数据库

1，获取以及商品分类信息

给一网页获取网页上商品信息的分类

using Skay.WebBot;

using System;

using System.Collections.Generic;

using System.ComponentModel;

using System.Data;

using System.Drawing;

using System.Linq;

using System.Text;

using System.Threading;

using System.Windows.Forms;

using Ivony.Html;

using Ivony.Html.Parser;

using System.Data.SqlClient;

namespace catchGoods

{

    public partial class Form1 : Form

    {

        public Form1()

        {

            InitializeComponent();

        }

        public static Thread th;

        private void button1_Click(object sender, EventArgs e)

        {

            th = new Thread(GetJDData);

            th.Start();

         }

        public void GetJDData()

        {

            SqlConnection conn = new SqlConnection("Data Source=.;Initial Catalog=StuTinafirst;User ID=sa;Password=123456");

            conn.Open();

            string str = "http://www.htluxe.com";

            HttpUtility http = new HttpUtility();

            string html = http.GetHtmlText(str);

            var documenthtml = new JumonyParser().Parse(html);

            var items = documenthtml.Find(".categroup dl");

            foreach(var item in items)

            {

                string name = item.FindFirst("h4 a").InnerText();

                string remarkOdd = item.FindFirst("h4 a").Attribute("href").Value();

                string remark = remarkOdd.Split('=')[];

                this.Invoke((EventHandler)(delegate

                {

                    listBox1.Items.Add(name);

                }));//有线程时listbox添加东西的时候要这么写不然报错谁知道什么鬼（委托？

                string into = string.Format("insert into exerciseOneSort (className, remark) values ('" + name + "', '" + remark + "')");

                SqlCommand com = new SqlCommand(into, conn);

                int i = com.ExecuteNonQuery();

                var elements = item.Find("dt p a");

                foreach(var element in elements)

                {

                    string nameTwo = element.InnerText();

                    string url = "http://www.htluxe.com/" + element.Attribute("href").Value();

                    string intoTwo = string.Format("insert into exerciseTwoSort (className, url, idplus) values ('" + nameTwo + "', '" + url + "', '" + remark + "')");

                    SqlCommand comTwo = new SqlCommand(intoTwo, conn);

                    int j = comTwo.ExecuteNonQuery();

                }

            }

        }

    }

}

完整版

using Skay.WebBot;

using System;

using System.Collections.Generic;

using System.ComponentModel;

using System.Data;

using System.Drawing;

using System.Linq;

using System.Text;

using System.Threading;

using System.Windows.Forms;

using Ivony.Html;

using Ivony.Html.Parser;

using System.Data.SqlClient;

using Newtonsoft.Json.Linq;

using Newtonsoft.Json;

namespace catchGoods

{

    public partial class Form1 : Form

    {

        public Form1()

        {

            InitializeComponent();

        }

        public static Thread th;

        private void button1_Click(object sender, EventArgs e)

        {

            th = new Thread(GetJDDataOne);

            th.Start();

            //SqlConnection conn = new SqlConnection("Data Source=.;Initial Catalog=StuTinafirst;User ID=sa;Password=123456");

            //conn.Open();

            //string str = string.Format("delete from exerciseTwoSort");

            //SqlCommand com = new SqlCommand(str, conn);

            //int w = com.ExecuteNonQuery();

         }

        public void GetJDDataOne()

        {

            SqlConnection conn = new SqlConnection("Data Source=.;Initial Catalog=StuTinafirst;User ID=sa;Password=123456");

            conn.Open();

            string str = "http://www.htluxe.com";

            HttpUtility http = new HttpUtility();

            string html = http.GetHtmlText(str);

            var documenthtml = new JumonyParser().Parse(html);

            var items = documenthtml.Find(".categroup dl");

            foreach(var item in items)

            {

                string name = item.FindFirst("h4 a").InnerText();

                string remarkOdd = item.FindFirst("h4 a").Attribute("href").Value();

                string remark = remarkOdd.Split('=')[];

                this.Invoke((EventHandler)(delegate

                {

                    listBox1.Items.Add(name+"  "+remark);

                }));//有线程时listbox添加东西的时候要这么写不然报错谁知道什么鬼

                string into = string.Format("insert into exerciseOneSort (className, remark) values ('" + name + "', '" + remark + "')");

                SqlCommand com = new SqlCommand(into, conn);

                int i = com.ExecuteNonQuery();

                var elements = item.Find("dt p a");

                foreach(var element in elements)

                {

                    string nameTwo = element.InnerText();

                    string url = "http://www.htluxe.com/" + element.Attribute("href").Value();

                    this.Invoke((EventHandler)(delegate

                    {

                        listBox1.Items.Add(nameTwo + "  " +url + "  " + remark);

                    }));//有线程时listbox添加东西的时候要这么写不然报错谁知道什么鬼

                    string intoTwo = string.Format("insert into exerciseTwoSort (className, url, idplus) values ('" + nameTwo + "', '" + url + "', '" + remark + "')");

                    SqlCommand comTwo = new SqlCommand(intoTwo, conn);

                    int j = comTwo.ExecuteNonQuery();

                }

            }

        }

        int page = ;

        string surl;

        public static Thread th2;

        private void button2_Click(object sender, EventArgs e)

        {

            listBox1.Items.Clear();

            th2 = new Thread(threadTwo);

            th2.Start();

            //SqlConnection conn = new SqlConnection("Data Source=.;Initial Catalog=StuTinafirst;User ID=sa;Password=123456");

            //conn.Open();

            //string str = string.Format("delete from GoodsList");

            //SqlCommand com = new SqlCommand(str, conn);

            //int d = com.ExecuteNonQuery();

            //MessageBox.Show(Convert.ToString(d));

        }

        public void threadTwo()

        {

            SqlConnection conn = new SqlConnection("Data Source=.;Initial Catalog=StuTinafirst;User ID=sa;Password=123456");

            conn.Open();

            //如果字符串中含有单引号，解决方法1----------------------------------

            //string titlestr = "念佛'夜晚访'问欧诺'法";

            //string pricestr = "99.00";

            //string sqlstr = string.Format("insert into goods (name,price) values (@name,'" + pricestr + "')");

            //SqlCommand sqlcom = new SqlCommand(sqlstr, conn);

            //sqlcom.Parameters.Add("@name", titlestr);

            //sqlcom.ExecuteNonQuery();

            //解决方法2-----------------------------------------------------------------------

            //string bufffuck = "fdgjjf'fgfgf";

            //bufffuck = bufffuck.Replace("'", "''");

            //string sqlstr = string.Format("insert into goods (name) values ('"+bufffuck+"')");

            //SqlCommand sqlcom = new SqlCommand(sqlstr, conn);

            //int y = sqlcom.ExecuteNonQuery();

            string sel = string.Format("select url from exerciseTwoSort");

            DataTable dt = new DataTable();

            SqlDataAdapter dapt = new SqlDataAdapter(sel, conn);

            dapt.Fill(dt);

            for (int i = ; i < dt.Rows.Count; i++)

            {

                surl = dt.Rows[i][].ToString();

                HttpUtility httpTwo = new HttpUtility();

                string htmlTwo = httpTwo.GetHtmlText(surl);

                var documenthtml = new JumonyParser().Parse(htmlTwo);

                var pageto = Convert.ToString(documenthtml.FindFirst(".goods-page-min label").InnerText());

                page = Convert.ToInt32(pageto.Split('/')[]);

                GetJDData();

            }

        }

         void GetJDData()

         {

              for (int j = ; j <= page; j++)

              {

                  string htmlTwo = surl + "&price_min=0&price_max=0&page=" + j + "&sort=sort_order%20asc,last_update&order=DESC";

                  HttpUtility httpMid = new HttpUtility();

            string htmlMid = httpMid.GetHtmlText(htmlTwo);

            var documenthtmlMid = new JumonyParser().Parse(htmlMid);

            var items = documenthtmlMid.Find(".piclist li");

            foreach(var item in items)

            {

                string title = item.FindFirst(".base a").InnerText();

                title = title.Replace("'", "''");

                //string goodsurl = "http://www.htluxe.com/"+item.FindFirst(".base a").Attribute("href").Value();

                //string subhtml = http.GetHtmlText(goodsurl, "utf-8", "text/html; charset=utf-8");

                //string Area_Html = http.GetHtmlText(goodsurl.Split('?')[0] + "?act=price&" + goodsurl.Split('?')[1], "utf-8", "text/html;charset=utf-8", "");

                try

                {

                    string nowPrice = item.FindFirst(".minprice").InnerText();

                    string oldPrice = item.FindFirst(".maxprice").InnerText();

                    string popular = item.FindFirst(".ratecount strong").InnerText();

                    string sales = item.FindFirst(".soldnum strong").InnerText();

                    string contents = item.FindFirst(".commentcount strong").InnerText().ToString();

                    string htmlThree = "http://www.htluxe.com/" + item.FindFirst("dt a").Attribute("href").Value().ToString();

                    HttpUtility httpThree = new HttpUtility();

                    string htmlBuff = httpThree.GetHtmlText(htmlThree);

                    var documenthtmlThree = new JumonyParser().Parse(htmlBuff);

                    string sben = documenthtmlThree.FindFirst(".promotionMiddleTop p").InnerText().ToString();

                    string num = sben.Split('：')[];

                       string starLevel = documenthtmlThree.FindFirst(".m-ratescore i").InnerText().ToString();

                       bufff(title, nowPrice, oldPrice, popular, sales, num, contents, starLevel);

                       this.Invoke((EventHandler)(delegate

                       {

                           listBox1.Items.Add(title + "  " + nowPrice + "  " + num + "  " + oldPrice + "  " + sales + "  " + popular + " " + contents + " " + starLevel);

                       }));

                        //有线程时listbox添加东西的时候要这么写不然报错谁知道什么鬼

                    //this.listBox1.Items.Add("");

                    //listBox1.Items.Add(title + "  " + nowPrice + "  " + num + "  " + oldPrice + "  " + sales + "  " + popular);                   

                }

                catch

                {

                    MessageBox.Show("异常");

                }

            }

              }

        }

        private static void bufff(string title, string nowPrice, string oldPrice,

            string popular, string sales, string num, string contents, string starLevel)

        {

            SqlConnection conn2 = new SqlConnection("Data Source=.;Initial Catalog=StuTinafirst;User ID=sa;Password=123456");

            conn2.Open();

            string strstr = string.Format("insert into GoodsList (name, num, sales, popular, starLevel, contents, price, oldPrice) values ('" + title + "', '" +num + "',  '" + sales + "', '" + popular + "', '"+starLevel+"', '"+contents+"', '" + nowPrice + "', '" + oldPrice + "')");

            SqlCommand com2 = new SqlCommand(strstr, conn2);

            int g = com2.ExecuteNonQuery();

        }

    }

}

C#获取网页信息并存入数据库的更多相关文章

C# HttpWebRequest 绝技根据URL地址获取网页信息
如果要使用中间的方法的话,可以访问我的帮助类完全免费开源:C# HttpHelper,帮助类,真正的Httprequest请求时无视编码,无视证书,无视Cookie,网页抓取 1.第一招,根据URL地 ...
使用URLConnection获取网页信息的基本流程
参考自core java v2, chapter3 Networking. 注:URLConnection的子类HttpURLConnection被广泛用于Android网络客户端编程,它与apach ...
使用URLConnection获取网页信息的基本流程分类： H1_ANDROID 2013-10-12 23:51 3646人阅读评论(0) 收藏
参考自core java v2, chapter3 Networking. 注:URLConnection的子类HttpURLConnection被广泛用于Android网络客户端编程,它与apach ...
C# 获取网页信息
获取网页源码 ///通过HttpWebResponse public string GetUrlHtml(string url) { string strHtml = string.Empty; Ht ...
Python 爬虫招聘信息并存入数据库
新学习了selenium,啪一下腾讯招聘 from lxml import etree from selenium import webdriver import pymysql def Geturl ...
C#获取网页信息核心方法（入门一）
目录:信息采集入门系列目录下面记录的是我自己整理的C#请求页面核心类,主要有如下几个方法 1.HttpWebRequest Get请求获得页面html 2.HttpWebRequest Post请求 ...
python爬虫爬取ip记录网站信息并存入数据库
import requests import re import pymysql #10页仔细观察路由 db = pymysql.connect("localhost",&quo ...
python获取网页信息的三种方法
import urllib.request import http.cookiejar url = 'http://www.baidu.com/' # 方法一 print('方法一') req_one ...
获取网页上数据（图片、文字、视频）－b
Demo地址:http://download.csdn.net/detail/u012881779/8831835 获取网页上所有图片.获取所有html.获取网页title.获取网页内容文字... . ...

随机推荐

Vue.js官方文档学习笔记(三)创建Vue实例
创建Vue实例每个 Vue 应用都是通过用 Vue 函数创建一个新的 Vue 实例开始的: var vm=new Vue({ //选项 }) Vue的设计受到了mvvm的启发当创建一个 Vue 实 ...
xml文件简单读取，循环遍历
xml文件的简单读取出来生成dom4j.Document对象树,循环遍历获取节点例子: import java.util.Iterator; import org.dom4j.Document; i ...
springBoot2.0 Yaml值获取
1. pom.xml添加如下依赖 <dependency> <groupId>org.springframework.boot</groupId> <arti ...
AtCoder Beginner Contest 076
A - Rating Goal Time limit : 2sec / Memory limit : 256MB Score : 100 points Problem Statement Takaha ...
P多行溢出省略号的处理
因为-webkit-line-clamp: 2不兼容火狐或IE,采用判断浏览器的方式来启用哪个方式先判断是什么浏览器 //判断是否是谷歌浏览器 if (!stripos($_SERVER[" ...
vue学习【四】vuex快速入门
大家好,我是一叶,今天我们继续踩坑.今天的内容是vuex快速入门,页面传值不多的话,不建议vuex,直接props进行父子间传值就行,使用vuex就显得比较臃肿. 我们先预览一下效果,如图1所示. 图 ...
join 与 countdownlatch 的区别扩展栅栏 CyclicBarrier
我们先看一个小例子 , 使用 join 与CountDownSlatch 都可以完成当1,2线程完全结束后 3 线程 start 对比我们就能够知道 CountDownSlatch 比 JOIN ...
Linux文件读写笔记
读文件: #include <stdio.h> #include <stdlib.h> #include <unistd.h> //linux下面的头文件 #inc ...
Hibernate 最简单实例
我从网上下载了 hibernate-release-4.3.0.Final.zip,解压缩,把/lib/required文件夹下的所有jar包加入到eclipse项目中的Referenced Libr ...
iconv_open - 分配一个字符集转换的描述符
总览 (SYNOPSIS) #include <iconv.h> iconv_t iconv_open (const char* tocode, const char* fromcode) ...

C#获取网页信息并存入数据库

C#获取网页信息并存入数据库的更多相关文章

随机推荐

热门专题