蜘蛛页面

from selenium import webdriver

import time

import random

from bs4 import *

import pymysql

h, pt, u, p, db = 'localhost', , 'root', 'root', 'test'

def mysql_fetch(sql, res_type='tuple'):

    global h, pt, u, p, db

    try:

        conn = pymysql.connect(host=h, port=pt, user=u, passwd=p, db=db, charset='utf8')

    except Exception as e:

        print(e)

        return ()

    if res_type == 'dic':

        cursor = conn.cursor(pymysql.cursors.DictCursor)

    else:

        cursor = conn.cursor()

    cursor.execute(sql)

    conn.commit()

    cursor.close()

    conn.close()

    return cursor.fetchall()

def mysql_write(sql):

    global h, pt, u, p, db

    try:

        conn = pymysql.connect(host=h, port=pt, user=u, passwd=p, db=db, charset='utf8')

    except Exception as e:

        print(e)

        return

    cursor = conn.cursor()

    cursor.execute(sql)

    conn.commit()

    cursor.close()

    conn.close()

    return 

browser = webdriver.Chrome()

f_url_l = ['https://www.baidu.com/', 'https://www.so.com/']

f_url_l_a = f_url_l[int(time.time()) % len(f_url_l)]

browser.get(f_url_l_a)

time.sleep(random.randint(, ))

url = 'https://so.gushiwen.org/shiwenv_5a36a4613434.aspx'

js = 'window.location.href="{}";'.format(url)

browser.execute_script(js)

# img_url = 'https://img.gushiwen.org/authorImg/daishulun.jpg'

myhtml = 'D:\\myhtml\\{}gushiwen.tmp.html'.format(random.randint(, ))

with open(myhtml, 'w', encoding='utf-8') as fw:

    fw.write(browser.page_source)

sql = 'INSERT INTO parent_url (page_title,page_url,children_url) VALUES '

with open(myhtml, 'r', encoding='utf-8') as myhtml_o:

    bs = BeautifulSoup(myhtml_o, 'html.parser')

    url_l = [i.attrs['href'] for i in bs.find_all('a')]

res_l = []

sql_l = []

for i in url_l:

    if 'gushiwen.org' in i and 'javascript' not in i and 'http' in i and i[:] == 'http':

        # sql_chk = 'SELECT * FROM  parent_url WHERE page_url="{}" AND children_url="{}"'.format(url, i)

        # r = mysql_fetch(sql_chk)

        # if len(r) > :

        #     continue

        if i not in res_l:

            if i == url:

                continue

            res_l.append(i)

            s = '("{}","{}","{}")'.format(browser.title, url, i)

            sql_l.append(s)

if len(sql_l) > :

    sql = '{}{}'.format(sql, ','.join(sql_l))

    print(sql)

    mysql_write(sql)

    sql_del = 'DELETE FROM parent_url WHERE id IN ( SELECT  min_id FROM ( SELECT  MIN(id) AS min_id, COUNT(1) AS c FROM parent_url GROUP BY CONCAT(page_url,children_url) ) AS tab  WHERE c>1 )'

    print(sql_del)

    mysql_write(sql_del)

while True:

    sql_ori = 'SELECT page_url,children_url FROM  parent_url WHERE if_spider=0 ORDER BY  id DESC '

    res = mysql_fetch(sql_ori, 'dic')

    for d in res:

        page_url, children_url = d['page_url'], d['children_url']

        url = children_url

        js = 'window.location.href="{}";'.format(url)

        browser.execute_script(js)

        time.sleep()

        browser.refresh()

        myhtml = 'D:\\myhtml\\{}gushiwen.tmp.html'.format(random.randint(, ))

        with open(myhtml, 'w', encoding='utf-8') as fw:

            fw.write(browser.page_source)

        sql = 'INSERT INTO parent_url (page_title,page_url,children_url) VALUES '

        with open(myhtml, 'r', encoding='utf-8') as myhtml_o:

            bs = BeautifulSoup(myhtml_o, 'html.parser')

            try:

                url_l = [i.attrs['href'] for i in bs.find_all('a')]

            except Exception as e:

                print(e)

                continue

        res_l = []

        sql_l = []

        for i in url_l:

            # /user/findpwd.aspx?from=http://so.gushiwen.org/user/collect.aspx

            if 'gushiwen.org' in i and 'javascript' not in i and 'http' in i and i[:] == 'http':

                # sql_chk = 'SELECT * FROM  parent_url WHERE page_url="{}" AND children_url="{}"'.format(url, i)

                # print(sql_chk)

                # r = mysql_fetch(sql_chk)

                # print(r)

                # if len(r) > :

                #     continue

                if i not in res_l:

                    if i == url:

                        continue

                    res_l.append(i)

                    s = '("{}","{}","{}")'.format(browser.title, url, i)

                    # sql_break = '{}{}'.format(sql, s)

                    # print(sql_break)

                    # mysql_write(sql_break)

                   # print(s)

                    sql_l.append(s)

        if len(sql_l) > :

            sql = '{}{}'.format(sql, ','.join(sql_l))

            print(sql)

            mysql_write(sql)

            sql_del = 'DELETE FROM parent_url WHERE id IN ( SELECT  min_id FROM ( SELECT  MIN(id) AS min_id, COUNT(1) AS c FROM parent_url GROUP BY CONCAT(page_url,children_url) ) AS tab  WHERE c>1 )'

            print(sql_del)

            mysql_write(sql_del)

            sql_udp = 'UPDATE parent_url SET if_spider=1 WHERE page_url="{}" AND children_url="{}"'.format(page_url,

                                                                                                           url)

            mysql_write(sql_udp)

            print(sql_udp)

    time.sleep()

dd =

CREATE TABLE `parent_url` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`page_title` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`page_url` text,
`children_url` text,
`if_spider` tinyint(4) DEFAULT '0',
PRIMARY KEY (`id`)
) ENGINE=MyISAM AUTO_INCREMENT=5328 DEFAULT CHARSET=latin1;

先写入，后删除

避免每个写入前的检查

消耗时间

获取一个网站的全部url

修复逻辑错误

支持 多进程   脚本多开

from selenium import webdriver

import time

import random

from bs4 import *

import pymysql

h, pt, u, p, db = 'localhost', 3306, 'root', 'root', 'test'

def mysql_fetch(sql, res_type='tuple'):

    global h, pt, u, p, db

    try:

        conn = pymysql.connect(host=h, port=pt, user=u, passwd=p, db=db, charset='utf8')

    except Exception as e:

        print(e)

        return ()

    if res_type == 'dic':

        cursor = conn.cursor(pymysql.cursors.DictCursor)

    else:

        cursor = conn.cursor()

    cursor.execute(sql)

    conn.commit()

    cursor.close()

    conn.close()

    return cursor.fetchall()

def mysql_write(sql):

    global h, pt, u, p, db

    try:

        conn = pymysql.connect(host=h, port=pt, user=u, passwd=p, db=db, charset='utf8')

    except Exception as e:

        print(e)

        return 1

    cursor = conn.cursor()

    cursor.execute(sql)

    conn.commit()

    cursor.close()

    conn.close()

    return 0

browser = webdriver.Chrome()

f_url_l = ['https://www.baidu.com/', 'https://www.so.com/']

f_url_l_a = f_url_l[int(time.time()) % len(f_url_l)]

browser.get(f_url_l_a)

time.sleep(random.randint(1, 2))

url = 'https://so.gushiwen.org/shiwenv_5a36a4613434.aspx'

js = 'window.location.href="{}";'.format(url)

browser.execute_script(js)

# img_url = 'https://img.gushiwen.org/authorImg/daishulun.jpg'

myhtml = 'D:\\myhtml\\{}gushiwen.tmp.html'.format(random.randint(123, 999))

with open(myhtml, 'w', encoding='utf-8') as fw:

    fw.write(browser.page_source)

sql = 'INSERT INTO parent_url (page_title,page_url,children_url) VALUES '

with open(myhtml, 'r', encoding='utf-8') as myhtml_o:

    bs = BeautifulSoup(myhtml_o, 'html.parser')

    url_l = [i.attrs['href'] for i in bs.find_all('a')]

res_l = []

sql_l = []

for i in url_l:

    if 'gushiwen.org' in i and 'javascript' not in i and 'http' in i and i[0:4] == 'http':

        # sql_chk = 'SELECT * FROM  parent_url WHERE page_url="{}" AND children_url="{}"'.format(url, i)

        # r = mysql_fetch(sql_chk)

        # if len(r) > 0:

        #     continue

        if i not in res_l:

            if i == url:

                continue

            res_l.append(i)

            s = '("{}","{}","{}")'.format(browser.title, url, i)

            sql_l.append(s)

if len(sql_l) > 0:

    sql = '{}{}'.format(sql, ','.join(sql_l))

    print(sql)

    mysql_write(sql)

    sql_del = 'DELETE FROM parent_url WHERE id IN ( SELECT  max_id FROM ( SELECT  MAX(id) AS max_id, COUNT(1) AS c FROM parent_url GROUP BY CONCAT(page_url,children_url) ) AS tab  WHERE c>1 )'

    print(sql_del)

    mysql_write(sql_del)

import random

# https://www.gushiwen.org/FileNotFound.htm?aspxerrorpath=/user/findpwd.aspx

# https://so.gushiwen.org/user/findpwd.aspx?from=http://so.gushiwen.org/user/collect.aspx

# https://so.gushiwen.org/app/

url_kw_filter_l = ['FileNotFound', 'findpwd', '/app/']

while True:

    sql_filter = "DELETE FROM parent_url WHERE id IN ( SELECT id FROM ( SELECT id FROM  parent_url  WHERE   INSTR(page_title,'密码')>0 OR  INSTR(UPPER(page_url),'PWD')>0 OR  INSTR(UPPER(children_url),'PWD')>0) AS t);"

    mysql_write(sql_filter)

    sql_ori = 'SELECT page_url,children_url FROM  parent_url WHERE if_spider=0'

    res = mysql_fetch(sql_ori, 'dic')

    jump_c, jump_s = 0, random.randint(0, max(0, len(res) - 10))

    for d in res:

        jump_c += 1

        if jump_c < jump_s:

            continue

        page_url, children_url = d['page_url'], d['children_url']

        url = children_url

        continue_ = False

        for fl in url_kw_filter_l:

            if fl in url:

                continue_ = True

                break

        if continue_:

            continue

        js = 'window.location.href="{}";'.format(url)

        browser.execute_script(js)

        time.sleep(1)

        browser.refresh()

        myhtml = 'D:\\myhtml\\{}gushiwen.tmp.html'.format(random.randint(123, 999))

        with open(myhtml, 'w', encoding='utf-8') as fw:

            fw.write(browser.page_source)

        sql = 'INSERT INTO parent_url (page_title,page_url,children_url) VALUES '

        with open(myhtml, 'r', encoding='utf-8') as myhtml_o:

            bs = BeautifulSoup(myhtml_o, 'html.parser')

            try:

                url_l = [i.attrs['href'] for i in bs.find_all('a')]

            except Exception as e:

                print(e)

                continue

        res_l = []

        sql_l = []

        for i in url_l:

            # /user/findpwd.aspx?from=http://so.gushiwen.org/user/collect.aspx

            if 'gushiwen.org' in i and 'javascript' not in i and 'http' in i and i[0:4] == 'http':

                # sql_chk = 'SELECT * FROM  parent_url WHERE page_url="{}" AND children_url="{}"'.format(url, i)

                # print(sql_chk)

                # r = mysql_fetch(sql_chk)

                # print(r)

                # if len(r) > 0:

                #     continue

                if i not in res_l:

                    if i == url:

                        continue

                    res_l.append(i)

                    s = '("{}","{}","{}")'.format(browser.title, url, i)

                    # sql_break = '{}{}'.format(sql, s)

                    # print(sql_break)

                    # mysql_write(sql_break)

                    # print(s)

                    sql_l.append(s)

        if len(sql_l) > 0:

            sql = '{}{}'.format(sql, ','.join(sql_l))

            print(sql)

            mysql_write(sql)

            sql_del = 'DELETE FROM parent_url WHERE id IN ( SELECT  max_id FROM ( SELECT  MAX(id) AS max_id, COUNT(1) AS c FROM parent_url GROUP BY CONCAT(page_url,children_url) ) AS tab  WHERE c>1 )'

            print(sql_del)

            mysql_write(sql_del)

            sql_udp = 'UPDATE parent_url SET if_spider=1 WHERE page_url="{}" AND children_url="{}"'.format(page_url,

                                                                                                           url)

            mysql_write(sql_udp)

            print(sql_udp)

time.sleep(3)

dd = 0

　　

代码的每一个功能点的模块化

from selenium import webdriver

import time

import random

from bs4 import *

import pymysql

h, pt, u, p, db = 'localhost', 3306, 'root', 'root', 'test'

def mysql_fetch(sql, res_type='tuple'):

    global h, pt, u, p, db

    try:

        conn = pymysql.connect(host=h, port=pt, user=u, passwd=p, db=db, charset='utf8')

    except Exception as e:

        print(e)

        return ()

    if res_type == 'dic':

        cursor = conn.cursor(pymysql.cursors.DictCursor)

    else:

        cursor = conn.cursor()

    cursor.execute(sql)

    conn.commit()

    cursor.close()

    conn.close()

    return cursor.fetchall()

def mysql_write(sql):

    global h, pt, u, p, db

    try:

        conn = pymysql.connect(host=h, port=pt, user=u, passwd=p, db=db, charset='utf8')

    except Exception as e:

        print(e)

        return 1

    cursor = conn.cursor()

    cursor.execute(sql)

    conn.commit()

    cursor.close()

    conn.close()

    return 0

browser = webdriver.Chrome()

f_url_l = ['https://www.baidu.com/', 'https://www.so.com/']

f_url_l_a = f_url_l[int(time.time()) % len(f_url_l)]

browser.get(f_url_l_a)

time.sleep(random.randint(1, 2))

url = 'https://so.gushiwen.org/shiwenv_5a36a4613434.aspx'

js = 'window.location.href="{}";'.format(url)

browser.execute_script(js)

# img_url = 'https://img.gushiwen.org/authorImg/daishulun.jpg'

myhtml = 'D:\\myhtml\\{}gushiwen.tmp.html'.format(random.randint(123, 999))

with open(myhtml, 'w', encoding='utf-8') as fw:

    fw.write(browser.page_source)

sql = 'INSERT INTO parent_url (page_title,page_url,children_url) VALUES '

with open(myhtml, 'r', encoding='utf-8') as myhtml_o:

    bs = BeautifulSoup(myhtml_o, 'html.parser')

    url_l = [i.attrs['href'] for i in bs.find_all('a')]

res_l = []

sql_l = []

for i in url_l:

    if 'gushiwen.org' in i and 'javascript' not in i and 'http' in i and i[0:4] == 'http':

        # sql_chk = 'SELECT * FROM  parent_url WHERE page_url="{}" AND children_url="{}"'.format(url, i)

        # r = mysql_fetch(sql_chk)

        # if len(r) > 0:

        #     continue

        if i not in res_l:

            if i == url:

                continue

            res_l.append(i)

            s = '("{}","{}","{}")'.format(browser.title, url, i)

            sql_l.append(s)

if len(sql_l) > 0:

    sql = '{}{}'.format(sql, ','.join(sql_l))

    print(sql)

    mysql_write(sql)

    sql_del = 'DELETE FROM parent_url WHERE id IN ( SELECT  max_id FROM ( SELECT  MAX(id) AS max_id, COUNT(1) AS c FROM parent_url GROUP BY CONCAT(page_url,children_url) ) AS tab  WHERE c>1 )'

    print(sql_del)

    mysql_write(sql_del)

import random

# https://www.gushiwen.org/FileNotFound.htm?aspxerrorpath=/user/findpwd.aspx

# https://so.gushiwen.org/user/findpwd.aspx?from=http://so.gushiwen.org/user/collect.aspx

# https://so.gushiwen.org/app/

# https://so.gushiwen.org/jiucuo.aspx?u=

url_kw_filter_l = ['FileNotFound', 'findpwd', '/app/', '/jiucuo.aspx']

sql_filter = "DELETE FROM parent_url WHERE id IN ( SELECT id FROM ( SELECT id FROM  parent_url  WHERE   INSTR(page_title,'密码')>0 OR  INSTR(UPPER(page_url),'PWD')>0 OR  INSTR(UPPER(children_url),'PWD')>0) AS t);"

sql_filter = "DELETE FROM parent_url WHERE id IN ( SELECT id FROM ( SELECT id FROM  parent_url  WHERE MYWHERE ) AS t);"

print(sql_filter)

sql_s_l = []

for i in url_kw_filter_l:

    ii = i.upper()

    s = "  INSTR(UPPER(page_url),'{}')>0  OR  INSTR(UPPER(children_url),'{}')>0 ".format(ii, ii)

    sql_s_l.append(s)

sql_filter = sql_filter.replace('MYWHERE', ' OR '.join(sql_s_l))

while True:

    mysql_write(sql_filter)

    sql_ori = 'SELECT page_url,children_url FROM  parent_url WHERE if_spider=0'

    res = mysql_fetch(sql_ori, 'dic')

    jump_c, jump_s = 0, random.randint(0, max(0, len(res) - 10))

    for d in res:

        jump_c += 1

        if jump_c < jump_s:

            continue

        page_url, children_url = d['page_url'], d['children_url']

        url = children_url

        js = 'window.location.href="{}";'.format(url)

        browser.execute_script(js)

        time.sleep(1)

        browser.refresh()

        myhtml = 'D:\\myhtml\\{}gushiwen.tmp.html'.format(random.randint(123, 999))

        with open(myhtml, 'w', encoding='utf-8') as fw:

            fw.write(browser.page_source)

        sql = 'INSERT INTO parent_url (page_title,page_url,children_url) VALUES '

        with open(myhtml, 'r', encoding='utf-8') as myhtml_o:

            bs = BeautifulSoup(myhtml_o, 'html.parser')

            try:

                url_l = [i.attrs['href'] for i in bs.find_all('a')]

            except Exception as e:

                print(e)

                continue

        res_l = []

        sql_l = []

        for i in url_l:

            # /user/findpwd.aspx?from=http://so.gushiwen.org/user/collect.aspx

            if 'gushiwen.org' in i and 'javascript' not in i and 'http' in i and i[0:4] == 'http':

                # sql_chk = 'SELECT * FROM  parent_url WHERE page_url="{}" AND children_url="{}"'.format(url, i)

                # print(sql_chk)

                # r = mysql_fetch(sql_chk)

                # print(r)

                # if len(r) > 0:

                #     continue

                if i not in res_l:

                    if i == url:

                        continue

                    res_l.append(i)

                    s = '("{}","{}","{}")'.format(browser.title, url, i)

                    # sql_break = '{}{}'.format(sql, s)

                    # print(sql_break)

                    # mysql_write(sql_break)

                    # print(s)

                    sql_l.append(s)

        if len(sql_l) > 0:

            sql = '{}{}'.format(sql, ','.join(sql_l))

            print(sql)

            mysql_write(sql)

            sql_del = 'DELETE FROM parent_url WHERE id IN ( SELECT  max_id FROM ( SELECT  MAX(id) AS max_id, COUNT(1) AS c FROM parent_url GROUP BY CONCAT(page_url,children_url) ) AS tab  WHERE c>1 )'

            print(sql_del)

            mysql_write(sql_del)

            sql_udp = 'UPDATE parent_url SET if_spider=1 WHERE page_url="{}" AND children_url="{}"'.format(page_url,

                                                                                                           url)

            mysql_write(sql_udp)

            print(sql_udp)

time.sleep(3)

dd = 0

　　

from selenium import webdriver

import time

import random

from bs4 import *

import pymysql

h, pt, u, p, db = 'localhost', 3306, 'root', 'root', 'test'

def mysql_fetch(sql, res_type='tuple'):

    global h, pt, u, p, db

    try:

        conn = pymysql.connect(host=h, port=pt, user=u, passwd=p, db=db, charset='utf8')

    except Exception as e:

        print(e)

        return ()

    if res_type == 'dic':

        cursor = conn.cursor(pymysql.cursors.DictCursor)

    else:

        cursor = conn.cursor()

    cursor.execute(sql)

    conn.commit()

    cursor.close()

    conn.close()

    return cursor.fetchall()

def mysql_write(sql):

    global h, pt, u, p, db

    try:

        conn = pymysql.connect(host=h, port=pt, user=u, passwd=p, db=db, charset='utf8')

    except Exception as e:

        print(e)

        return 1

    cursor = conn.cursor()

    cursor.execute(sql)

    conn.commit()

    cursor.close()

    conn.close()

    return 0

browser = webdriver.Chrome()

f_url_l = ['https://www.baidu.com/', 'https://www.so.com/']

f_url_l_a = f_url_l[int(time.time()) % len(f_url_l)]

browser.get(f_url_l_a)

time.sleep(random.randint(1, 2))

url = 'https://so.gushiwen.org/shiwenv_5a36a4613434.aspx'

js = 'window.location.href="{}";'.format(url)

browser.execute_script(js)

# img_url = 'https://img.gushiwen.org/authorImg/daishulun.jpg'

myhtml = 'D:\\myhtml\\{}gushiwen.tmp.html'.format(random.randint(123, 999))

with open(myhtml, 'w', encoding='utf-8') as fw:

    fw.write(browser.page_source)

sql = 'INSERT INTO parent_url (page_title,page_url,children_url) VALUES '

with open(myhtml, 'r', encoding='utf-8') as myhtml_o:

    bs = BeautifulSoup(myhtml_o, 'html.parser')

    url_l = [i.attrs['href'] for i in bs.find_all('a')]

res_l = []

sql_l = []

for i in url_l:

    if 'gushiwen.org' in i and 'javascript' not in i and 'http' in i and i[0:4] == 'http':

        # sql_chk = 'SELECT * FROM  parent_url WHERE page_url="{}" AND children_url="{}"'.format(url, i)

        # r = mysql_fetch(sql_chk)

        # if len(r) > 0:

        #     continue

        if i not in res_l:

            if i == url:

                continue

            res_l.append(i)

            s = '("{}","{}","{}")'.format(browser.title, url, i)

            sql_l.append(s)

if len(sql_l) > 0:

    sql = '{}{}'.format(sql, ','.join(sql_l))

    print(sql)

    mysql_write(sql)

    sql_del = 'DELETE FROM parent_url WHERE id IN ( SELECT  max_id FROM ( SELECT  MAX(id) AS max_id, COUNT(1) AS c FROM parent_url GROUP BY CONCAT(page_url,children_url) ) AS tab  WHERE c>1 )'

    print(sql_del)

    mysql_write(sql_del)

import random

# https://www.gushiwen.org/FileNotFound.htm?aspxerrorpath=/user/findpwd.aspx

# https://so.gushiwen.org/user/findpwd.aspx?from=http://so.gushiwen.org/user/collect.aspx

# https://so.gushiwen.org/app/

# https://so.gushiwen.org/jiucuo.aspx?u=

url_kw_filter_l = ['FileNotFound', 'findpwd', '/app/', '/jiucuo.aspx']

sql_filter = "DELETE FROM parent_url WHERE id IN ( SELECT id FROM ( SELECT id FROM  parent_url  WHERE   INSTR(page_title,'密码')>0 OR  INSTR(UPPER(page_url),'PWD')>0 OR  INSTR(UPPER(children_url),'PWD')>0) AS t);"

sql_filter = "DELETE FROM parent_url WHERE id IN ( SELECT id FROM ( SELECT id FROM  parent_url  WHERE MYWHERE ) AS t);"

sql_s_l = []

for i in url_kw_filter_l:

    ii = i.upper()

    s = "  INSTR(UPPER(page_url),'{}')>0  OR  INSTR(UPPER(children_url),'{}')>0 ".format(ii, ii)

    sql_s_l.append(s)

sql_filter = sql_filter.replace('MYWHERE', ' OR '.join(sql_s_l))

while True:

    mysql_write(sql_filter)

    print(sql_filter)

    sql_del = 'DELETE FROM parent_url WHERE id IN ( SELECT  max_id FROM ( SELECT  MAX(id) AS max_id, COUNT(1) AS c FROM parent_url GROUP BY CONCAT(page_url,children_url) ) AS tab  WHERE c>1 )'

    print(sql_del)

    mysql_write(sql_del)

    sql_pass = 'SELECT DISTINCT(page_url) FROM parent_url'

    url_pass = ['"{}"'.format(i[0]) for i in mysql_fetch(sql_pass, res_type='tuple')]

    # 乐观代码

    sql_ori = 'SELECT page_url,children_url FROM  parent_url WHERE if_spider=0 AND children_url NOT IN ({})'.format(

        ','.join(url_pass))

    res = mysql_fetch(sql_ori, 'dic')

    jump_c, jump_s = 0, random.randint(0, max(0, len(res) - 10))

    for d in res:

        jump_c += 1

        if jump_c < jump_s:

            continue

        page_url, children_url = d['page_url'], d['children_url']

        url = children_url

        js = 'window.location.href="{}";'.format(url)

        browser.execute_script(js)

        # time.sleep(1)

        # browser.refresh()

        try:

            for isc in range(1):

                time.sleep(1)

                js = 'window.scrollTo(0,document.body.scrollHeight)'

                browser.execute_script(js)

        except Exception as e:

            print('window.scrollTo-->', e)

        myhtml = 'D:\\myhtml\\{}gushiwen.tmp.html'.format(random.randint(123, 999))

        with open(myhtml, 'w', encoding='utf-8') as fw:

            fw.write(browser.page_source)

        sql = 'INSERT INTO parent_url (page_title,page_url,children_url) VALUES '

        with open(myhtml, 'r', encoding='utf-8') as myhtml_o:

            bs = BeautifulSoup(myhtml_o, 'html.parser')

            try:

                url_l = [i.attrs['href'] for i in bs.find_all('a')]

            except Exception as e:

                print(e)

                continue

        res_l = []

        sql_l = []

        for i in url_l:

            # /user/findpwd.aspx?from=http://so.gushiwen.org/user/collect.aspx

            if 'gushiwen.org' in i and 'javascript' not in i and 'http' in i and i[0:4] == 'http':

                # sql_chk = 'SELECT * FROM  parent_url WHERE page_url="{}" AND children_url="{}"'.format(url, i)

                # print(sql_chk)

                # r = mysql_fetch(sql_chk)

                # print(r)

                # if len(r) > 0:

                #     continue

                if i not in res_l:

                    if i == url:

                        continue

                    continue_ = False

                    for fi in url_kw_filter_l:

                        ii = fi.upper()

                        if fi in i.upper():

                            continue_ = True

                            break

                    if continue_:

                        continue

                    res_l.append(i)

                    s = '("{}","{}","{}")'.format(browser.title, url, i)

                    # sql_break = '{}{}'.format(sql, s)

                    # print(sql_break)

                    # mysql_write(sql_break)

                    # print(s)

                    sql_l.append(s)

        if len(sql_l) > 0:

            sql = '{}{}'.format(sql, ','.join(sql_l))

            print(sql)

            mysql_write(sql)

            sql_udp = 'UPDATE parent_url SET if_spider=1 WHERE page_url="{}" AND children_url="{}"'.format(page_url,

                                                                                                           url)

            mysql_write(sql_udp)

            print(sql_udp)

dd = 0

　　

from selenium import webdriver

import time

import random

from bs4 import *

import pymysql

h, pt, u, p, db = 'localhost', 3306, 'root', 'root', 'test'

def mysql_fetch(sql, res_type='tuple'):

    global h, pt, u, p, db

    try:

        conn = pymysql.connect(host=h, port=pt, user=u, passwd=p, db=db, charset='utf8')

    except Exception as e:

        print(e)

        return ()

    if res_type == 'dic':

        cursor = conn.cursor(pymysql.cursors.DictCursor)

    else:

        cursor = conn.cursor()

    cursor.execute(sql)

    conn.commit()

    cursor.close()

    conn.close()

    return cursor.fetchall()

def mysql_write(sql):

    global h, pt, u, p, db

    try:

        conn = pymysql.connect(host=h, port=pt, user=u, passwd=p, db=db, charset='utf8')

    except Exception as e:

        print(e)

        return 1

    cursor = conn.cursor()

    cursor.execute(sql)

    conn.commit()

    cursor.close()

    conn.close()

    return 0

browser = webdriver.Chrome()

f_url_l = ['https://www.baidu.com/', 'https://www.so.com/']

f_url_l_a = f_url_l[int(time.time()) % len(f_url_l)]

browser.get(f_url_l_a)

time.sleep(random.randint(1, 2))

url = 'https://so.gushiwen.org/shiwenv_5a36a4613434.aspx'

js = 'window.location.href="{}";'.format(url)

browser.execute_script(js)

# img_url = 'https://img.gushiwen.org/authorImg/daishulun.jpg'

myhtml = 'D:\\myhtml\\{}gushiwen.tmp.html'.format(random.randint(123, 999))

with open(myhtml, 'w', encoding='utf-8') as fw:

    fw.write(browser.page_source)

sql = 'INSERT INTO parent_url (page_title,page_url,children_url) VALUES '

with open(myhtml, 'r', encoding='utf-8') as myhtml_o:

    bs = BeautifulSoup(myhtml_o, 'html.parser')

    url_l = [i.attrs['href'] for i in bs.find_all('a')]

res_l = []

sql_l = []

for i in url_l:

    break

    if 'gushiwen.org' in i and 'javascript' not in i and 'http' in i and i[0:4] == 'http':

        # sql_chk = 'SELECT * FROM  parent_url WHERE page_url="{}" AND children_url="{}"'.format(url, i)

        # r = mysql_fetch(sql_chk)

        # if len(r) > 0:

        #     continue

        if i not in res_l:

            if i == url:

                continue

            res_l.append(i)

            s = '("{}","{}","{}")'.format(browser.title, url, i)

            sql_l.append(s)

if len(sql_l) > 0:

    sql = '{}{}'.format(sql, ','.join(sql_l))

    print(sql)

    mysql_write(sql)

    sql_del = 'DELETE FROM parent_url WHERE id IN ( SELECT  max_id FROM ( SELECT  MAX(id) AS max_id, COUNT(1) AS c FROM parent_url GROUP BY CONCAT(page_url,children_url) ) AS tab  WHERE c>1 )'

    print(sql_del)

    mysql_write(sql_del)

import random

# https://www.gushiwen.org/FileNotFound.htm?aspxerrorpath=/user/findpwd.aspx

# https://so.gushiwen.org/user/findpwd.aspx?from=http://so.gushiwen.org/user/collect.aspx

# https://so.gushiwen.org/app/

# https://so.gushiwen.org/jiucuo.aspx?u=

url_kw_filter_l = ['FileNotFound', 'findpwd', '/app/', '/jiucuo.aspx']

sql_filter = "DELETE FROM parent_url WHERE id IN ( SELECT id FROM ( SELECT id FROM  parent_url  WHERE   INSTR(page_title,'密码')>0 OR  INSTR(UPPER(page_url),'PWD')>0 OR  INSTR(UPPER(children_url),'PWD')>0) AS t);"

sql_filter = "DELETE FROM parent_url WHERE id IN ( SELECT id FROM ( SELECT id FROM  parent_url  WHERE MYWHERE ) AS t);"

sql_s_l = []

for i in url_kw_filter_l:

    ii = i.upper()

    s = "  INSTR(UPPER(page_url),'{}')>0  OR  INSTR(UPPER(children_url),'{}')>0 ".format(ii, ii)

    sql_s_l.append(s)

sql_filter = sql_filter.replace('MYWHERE', ' OR '.join(sql_s_l))

while True:

    mysql_write(sql_filter)

    print(sql_filter)

    sql_del = 'DELETE FROM parent_url WHERE id IN ( SELECT  max_id FROM ( SELECT  MAX(id) AS max_id, COUNT(1) AS c FROM parent_url GROUP BY CONCAT(page_url,children_url) ) AS tab  WHERE c>1 )'

    print(sql_del)

    mysql_write(sql_del)

    sql_pass = 'SELECT DISTINCT(page_url) FROM parent_url'

    url_pass = ['"{}"'.format(i[0]) for i in mysql_fetch(sql_pass, res_type='tuple')]

    # 乐观代码

    sql_ori = 'SELECT page_url,children_url FROM  parent_url WHERE if_spider=0 AND children_url NOT IN ({})'.format(

        ','.join(url_pass))

    res = mysql_fetch(sql_ori, 'dic')

    jump_c, jump_s = 0, random.randint(0, max(0, len(res) - 10))

    for d in res:

        jump_c += 1

        if jump_c < jump_s:

            continue

        page_url, children_url = d['page_url'], d['children_url']

        url = children_url

        js = 'window.location.href="{}";'.format(url)

        browser.execute_script(js)

        # time.sleep(1)

        # browser.refresh()

        try:

            for isc in range(1):

                time.sleep(1)

                js = 'window.scrollTo(0,document.body.scrollHeight)'

                browser.execute_script(js)

        except Exception as e:

            print('window.scrollTo-->', e)

        myhtml = 'D:\\myhtml\\{}gushiwen.tmp.html'.format(random.randint(123, 999))

        with open(myhtml, 'w', encoding='utf-8') as fw:

            fw.write(browser.page_source)

        sql = 'INSERT INTO parent_url (page_title,page_url,children_url) VALUES '

        with open(myhtml, 'r', encoding='utf-8') as myhtml_o:

            bs = BeautifulSoup(myhtml_o, 'html.parser')

            try:

                url_l = [i.attrs['href'] for i in bs.find_all('a')]

            except Exception as e:

                print(e)

                continue

        res_l = []

        sql_l = []

        for i in url_l:

            # /user/findpwd.aspx?from=http://so.gushiwen.org/user/collect.aspx

            if 'gushiwen.org' in i and 'javascript' not in i and 'http' in i and i[0:4] == 'http':

                # sql_chk = 'SELECT * FROM  parent_url WHERE page_url="{}" AND children_url="{}"'.format(url, i)

                # print(sql_chk)

                # r = mysql_fetch(sql_chk)

                # print(r)

                # if len(r) > 0:

                #     continue

                if i not in res_l:

                    if i == url:

                        continue

                    continue_ = False

                    for fi in url_kw_filter_l:

                        ii = fi.upper()

                        if fi in i.upper():

                            continue_ = True

                            break

                    if continue_:

                        continue

                    res_l.append(i)

                    s = '("{}","{}","{}")'.format(browser.title, url, i)

                    # sql_break = '{}{}'.format(sql, s)

                    # print(sql_break)

                    # mysql_write(sql_break)

                    # print(s)

                    sql_l.append(s)

        if len(sql_l) > 0:

            sql = '{}{}'.format(sql, ','.join(sql_l))

            print(sql)

            mysql_write(sql)

            sql_udp = 'UPDATE parent_url SET if_spider=1 WHERE page_url="{}" AND children_url="{}"'.format(page_url,

                                                                                                           url)

            mysql_write(sql_udp)

            print(sql_udp)

dd = 0

　　

(父,子)url有序二元组

蜘蛛页面 获取一个网站的全部url 乐观代码的更多相关文章

  1. jsp页面获取参数的方法（url解析、el表达式赋值、session取值）【原创】

    最近使用myEclispse做网站,使用jsp+js+css做页面,网站中常用到从列表进入详情页面的跳转,下面对详情页面的值填充方式做一个简单总结: 1.url中使用request获取参数 jsp上方 ...

  2. 用JavaScript获取一个超链接的绝对URL地址

    对于Web程序员来说,处理简单的URL格式也许会成为一场噩梦.试想一下,一个网址里有很多组成部分都会影响你对它的解析方法: 是否以/字符开头 是否以//开头 是否以?号开头 是否以#号开头 …等等 当 ...

  3. 通过cookies跳过验证码登陆页面，直接访问网站的其它URL

    我每次手动访问去NN网的一家酒店,就不需要登陆,一旦我用脚本打开就会让我登陆,而登陆页面又有验证码,不想识别验证码,所以就想:“通过cookies跳过验证码登陆页面,直接访问网站的其它URL”   转 ...

  4. Python 网络爬虫 009 (编程) 通过正则表达式来获取一个网页中的所有的URL链接，并下载这些URL链接的源代码

    通过 正则表达式 来获取一个网页中的所有的 URL链接,并下载这些 URL链接 的源代码 使用的系统:Windows 10 64位 Python 语言版本:Python 2.7.10 V 使用的编程 ...

  5. songtaste网站歌曲真实URL获取

    个人挺喜欢songtaste网站的歌曲的,下载方法也层出不穷,可是作为程序员如果不知其中原理的方法真是羞愧.首先简单点的方法当然有google插件这样的嗅探器了,不过这种工具的原理还不是很了解.今天先 ...

  6. 获取一个 app 的 URL Scheme 的方法：

    获取一个 app 的 URL Scheme 的方法: 上这个网站 URL Schemes 查一下相应的 app 的 URL Scheme 是否有被收录 第一种方法没找到的话,把相应的 app 的 ip ...

  7. ASP.NET 获取来源网站的网址,获取上一网页的网址,获取来源网页的URL,获取上一网页的URL

    ASP.NET 获取来源网站的网址,获取上一网页的网址,获取来源网页的URL, 获取上一网页的URL Uri Url = HttpContext.Current.Request.UrlReferrer ...

  8. 多域名环境，页面获取url的一种方案

    因为系统是分布式部署的.而且有多个域名,所以常常涉及到获取url的问题. 这是系统框架层面须要提供的能力.否则每一个模块都须要自己去想办法获取ip,就会非常混乱.上线也easy发生bug 主要须要解决 ...

  9. 通过Iframe在A网站页面内嵌入空白页面的方式，跨域获取B网站的数据返回给A网站！

    以下代码只是为演示该方法具体是如何操作的,实际的意义并不大. 其实这个方法还可以解决很多方面的跨域操作,以下两点为我工作中遇到的情况! 比如A系统中打开B系统页面的时候,获取B系统页面高度,A系统中可 ...

随机推荐

  1. python send email

    #!/usr/bin/python # -*- coding: UTF-8 -*- # coding:utf8 from smtplib import SMTP_SSL from email.head ...

  2. vue -vantUI tab切换时 list组件不触发load事件解决办法

    最近由于公司项目需要,用vue写了几个简单的页面.用到了vantUI List 列表 瀑布流滚动加载,用于控制长列表的展示 当列表即将滚动到底部时,会触发事件并加载更多列表项. (页面加载完成后默认会 ...

  3. JS 水印图片合成实例页面

    CSS代码: .clip { position: absolute; clip: rect(0 0 0 0); } HTML代码: <input type="file" id ...

  4. 两种js下载文件的方法（转）

    function DownURL(strRemoteURL, strLocalURL){ try{ var xmlHTTP = new ActiveXObject("Microsoft.XM ...

  5. Centos下PHP7.1打开Oracle扩展

    背景 最近因为项目研究了一下PHP集成Oracle,虽然这个组合很奇葩,但万一用到请勿采坑~ 环境 CentOS Linux release 7.3.1611 (Core) PHP7.1.1 安装or ...

  6. Python之协程函数

    Python之协程函数 什么是协程函数:如果一个函数内部yield的使用方法是表达式形式的话,如x=yield,那么该函数成为协程函数. def eater(name): print('%s star ...

  7. Hdu 5274 Dylans loves tree (树链剖分模板)

    Hdu 5274 Dylans loves tree (树链剖分模板) 题目传送门 #include <queue> #include <cmath> #include < ...

  8. python面向对象编程实例

    1.编写程序, 编写一个学生类, 要求有一个计数器的属性, 统计总共实例化了多少个学生 class Student: """学生类""" c ...

  9. ASP.NET MVC的帮助类HtmlHelper和UrlHelper

    在ASP.NET MVC框架中没有了自己的控件,页面显示完全就回到了写html代码的年代.还好在asp.net mvc框架中也有自带的HtmlHelper和UrlHelper两个帮助类.另外在MvcC ...

  10. CentOS 7 & Chinese Fonts bug

    CentOS 7 & Chinese Fonts bug # check $ yum grouplist $ yum grouplist hidden # root $ yum groupin ...