urllib

urllib

Python 标准库 urllib2 的使用细节

python 2.x

3.x

from urllib import request

with request.urlopen('https://api.douban.com/v2/book/2129650') as f:
    data = f.read()
    print('Status:', f.status, f.reason)
    for k, v in f.getheaders():
        print('%s: %s' % (k, v))
    print('Data:', data.decode('utf-8'))

模拟浏览器

from urllib import request

req = request.Request('http://www.douban.com/')
req.add_header('User-Agent', 'Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25')
with request.urlopen(req) as f:
    print('Status:', f.status, f.reason)
    for k, v in f.getheaders():
        print('%s: %s' % (k, v))
    print('Data:', f.read().decode('utf-8'))

兼容Python2.x 3.x

#!/usr/bin/python
#-*-coding:utf-8-*- 

import json
import os
import re

try:
    from urllib.parse import urlencode
    from urllib.request import Request, urlopen
except:
    from urllib import urlencode
    from urllib2 import Request, urlopen

def get_token():
    req = Request("%s/authorizations/user_token/%s" % (os.environ.get('JPY_HUB_API_URL'), os.environ.get('JPY_USER')))
    req.add_header('Authorization', "token %s" % os.environ.get('JPY_HUB_API_TOKEN'))
    req.add_header("Host", "192.168.176.100")
    req.add_header("Referer", "%s/authorizations/user_token" % os.environ.get('JPY_HUB_API_URL'))

    data_success = None
    data_error = None

    try:
        f = urlopen(req)
    except:
        print('fail to open')
        exit(-1)

    try:
        data_success = f.read()
    except:
        data_error = {"user_token": ""}
    finally:
        f.close()

    if not data_error is None:
        data = data_error
    else:
        data = json.loads(data_success.decode('utf-8'))

    return data["user_token"]

def validate_year(v):
    pattern = re.compile(r'^\d{4}$')

    match = pattern.match(str(v))

    if match:
        return True
    else:
        return False

def get_data(url):
    req = Request(url)
    req.add_header('Authorization', "Bearer %s" % get_token())
    data_success = None
    data_error = None

    try:
        f = urlopen(req)
    except:
        print('fail to open')
        exit(-1)

    try:
        data_success = f.read()
    except:
        data_error = {"errmsg": "请求发生错误"}
    finally:
        f.close()

    if not data_error is None:
        data = data_error
    else:
        data = json.loads(data_success.decode('utf-8'))

    return data

def get_market_list():
    url = 'https://xxx.xx.xxx/quote/v1/market/list'
    #url = 'http://127.0.0.1:8000/json/market_list.json'

    return get_data(url)

def get_market_holiday(finance_mic, date=None):
    mic = ('O', 'N', 'A', 'HK', 'DCE', 'SHFE', 'SZ', 'SS', 'CZCE')
    if not finance_mic in mic:
        return {"errmsg": "参数错误"}

    if not date is None and \
        not validate_year(date):
        return {"errmsg": "参数错误"}

    search = urlencode([
        ('finance_mic', finance_mic),
        ('date', date)
    ])

    url = 'https://xxxx.xx.xxx/quote/v1/market/holiday?%s' % search

    return get_data(url)