#!/usr/env  python
#-*- coding: utf-8 -*-
import urllib
import urllib2
import random
import requests
import os,sys
import MySQLdb
from sgmllib import SGMLParser
from BeautifulSoup import BeautifulSoup
import re
num=0
def main():
try:
conn=MySQLdb.connect(host='localhost',user='root',passwd='123456',db='googlemarket',charset="utf8")
conn.query("set names utf8")
except Exception,e:
print e
sys.exit()
cursor=conn.cursor()
category=['PERSONALIZATION','TRANSPORTATION','SPORTS','HEALTH_AND_FITNESS','APP_WALLPAPER','COMICS','MEDICAL','BUSINESS','BOOKS_AND_REFERENCE','WEATHER','ENTERTAINMENT','MEDIA_AND_VIDEO','APP_WIDGETS','TOOLS','PHOTOGRAPHY','PRODUCTIVITY','EDUCATION','NEWS_AND_MAGAZINES','TRAVEL_AND_LOCAL','LIFESTYLE','SOCIAL','FINANCE','SHOPPING','LIBRARIES_AND_DEMO','COMMUNICATION','MUSIC_AND_AUDIO','GAME']
for k in range(0,27):
t="https://play.google.com/store/apps/category/"+category[k]
html=requests.get(t)
preresult=html.content
soup=BeautifulSoup(preresult)
result=soup.prettify("utf-8")
pattern=re.compile('<a class="title" href="(.+?)" title')
dataresult=re.findall(pattern,result)
dataresult=list(set(dataresult))
for i in dataresult:
url="https://play.google.com"+i
print url
#url="https://play.google.com/store/apps/details?id=com.androidesk&hl=zh_CNhttps%3A%2F%2Fplay.google.com%2Fstore%2Fapps%2Fdetails%3Fid%3Dcom.androidesk" html=requests.get(url)
preresult=html.content
soup=BeautifulSoup(preresult)
result=soup.prettify("utf-8")
#名称
pattern=re.compile('<div class="document-title" itemprop="name">[\s\S]*?<div>([\s\S]*?)</div>')
data0=re.findall(pattern,result)
for items in data0:
print items
#制造商
pattern=re.compile('itemprop="name">([\s\S]*?)</a>')
data1=re.findall(pattern,result) make=data1[0].split("\n") print make[8]
#版本
pattern=re.compile('itemprop="softwareVersion">([\s\S]*?)</div>')
data2=re.findall(pattern,result)
print data2[0]
#更新时间
pattern=re.compile('itemprop="datePublished">([\s\S]*?)</div>')
data3=re.findall(pattern,result)
print data3[0]
#文件大小
pattern=re.compile('itemprop="fileSize">([\s\S]*?)</div>')
data4=re.findall(pattern,result)
print data4[0]
#支持固件
pattern=re.compile('itemprop="operatingSystems">([\s\S]*?)</div>')
data5=re.findall(pattern,result)
print data5[0]
#说明
pattern=re.compile('itemprop="description">[\s\S]*?<div>([\s\S]*?)</div>')
data6=re.findall(pattern,result)
for items in data6:
print re.sub('[<br /> <p> </p>]',' ',items)
sql="insert into address(name,version,developer,pubtime,filesize,support,introduction) values(%s,%s,%s,%s,%s,%s,%s)"
for items in data6: if(data5):
#values=(data0[0],data1[0],data2[0],data3[0],data4[0],data5[0],re.sub('<br />',' ',items))
#else:
#values=(data0[0],data1[0],data2[0],data3[0],data4[0],'NULL',re.sub('<br />',' ',items))
#print values
#print sql % values
#cursor.execute(sql,values)
#conn.commit()
pattern=re.compile('<img class="cover-image" src=(.+?) alt="Cover art" itemprop="image" />')
data=re.findall(pattern,result)
global num
for j in data:
print j
print type(j)
headers = {'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201'}
temp=requests.get(j[1:-2], headers=headers)
f=file("googlemarket/"+str(num),"w+")
num=num+1
print num
f.write(temp.content) if __name__=="__main__":
main()

<type 'str'>
Traceback (most recent call last):
  File "crawler0729.py", line 103, in <module>
    main()
  File "crawler0729.py", line 91, in main
    temp=requests.get(j[1:-2], headers=headers)
  File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 55, in get
    return request('get', url, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 44, in request
    return session.request(method=method, url=url, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 335, in request
    resp = self.send(prep, **send_kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 438, in send
    r = adapter.send(request, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/adapters.py", line 327, in send
    raise ConnectionError(e)
requests.exceptions.ConnectionError: HTTPSConnectionPool(host='lh3.ggpht.com', port=443): Max retries exceeded with url: /RBld17rLw4Ik0JtOaKk4bZB2RiGJ2R8H5Q8Rjw3Hh6BAM694fOzzKj1TJFr7R02ZS_40=w30 (Caused by <class 'socket.error'>: [Errno 101] Network is unreachable)

python google play的更多相关文章

  1. Python+Google Geocoding

    本文主要介绍使用Python调用Google Geocoding API进行地址到地理坐标的转换. Google Geocoding参考https://developers.google.com/ma ...

  2. 吴裕雄--天生自然python Google深度学习框架:Tensorflow实现迁移学习

    import glob import os.path import numpy as np import tensorflow as tf from tensorflow.python.platfor ...

  3. 详解Python Google Protocol Buffer

    为什么要使用PB? PB(Protocol Buffer)是 Google 开发的用于结构化数据交换格式,作为腾讯云日志服务标准写入格式.因此用于写入日志数据前,需要将日志原始数据序列化为 PB 数据 ...

  4. 吴裕雄--天生自然python Google深度学习框架:经典卷积神经网络模型

    import tensorflow as tf INPUT_NODE = 784 OUTPUT_NODE = 10 IMAGE_SIZE = 28 NUM_CHANNELS = 1 NUM_LABEL ...

  5. 吴裕雄--天生自然python Google深度学习框架:图像识别与卷积神经网络

  6. 吴裕雄--天生自然python Google深度学习框架:MNIST数字识别问题

    import tensorflow as tf from tensorflow.examples.tutorials.mnist import input_data INPUT_NODE = 784 ...

  7. 吴裕雄--天生自然python Google深度学习框架:深度学习与深层神经网络

  8. 吴裕雄--天生自然python Google深度学习框架:TensorFlow实现神经网络

    http://playground.tensorflow.org/

  9. 吴裕雄--天生自然python Google深度学习框架:Tensorflow基础应用

    import tensorflow as tf a = tf.constant([1.0, 2.0], name="a") b = tf.constant([2.0, 3.0], ...

随机推荐

  1. 利用XPath解析带有xmlns的XML文件

    在.net中,编写读取xml 的程序中提示"未将对象引用设置到对象的实例",当时一看觉得有点奇怪.为什么在读取xml数据的时候也要实例化一个对象.google了才知道,xml文件中 ...

  2. 从网页psd到html的开发

    从网上下载了一张psd的网页设计稿,初学html+css,所以记录一下我的学习过程.原图是这个样子:             原图                                   ...

  3. asp.net中的<%%>形式的详细用法实例讲解

    asp.net中的代码分离模式我们肯定都不陌生,C#(或者其它语言)写的代码一般不会和设计语言HTML混在一起,但是有的时候也避免不了,这时就会在UI页面里用<%%>来绑定显示.绑定变量数 ...

  4. iOS+Swift: 使用MessageUI.framework发送短信

    在iOS中, 可以使用MessageUI.framework框架发送短信, 步骤如下: 代码下载http://git.oschina.net/yao_yu/swift_cnblogs_samples/ ...

  5. 自动发布工具版本从python2升级成python3后遇到的种种问题(涉及paramiko,Crypto,zipfile等等)

    从在公司实习到正式入职,一直还在被同事使用的是我写的一个自动发布工具.该工具的主要功能是:开发人员给出需要更新的代码包(zip格式),测试人员将该代码包部署到测服,这些代码包和JIRA数据库里的项目信 ...

  6. (转) 基于Theano的深度学习(Deep Learning)框架Keras学习随笔-01-FAQ

    特别棒的一篇文章,仍不住转一下,留着以后需要时阅读 基于Theano的深度学习(Deep Learning)框架Keras学习随笔-01-FAQ

  7. NHibernate configuration

    http://blog.csdn.net/dbcolor/article/details/2061929

  8. python判断用户注册中用户名是否包含非法字符

    class UserRegisterForm(ModelForm): role = forms.IntegerField() check_password = forms.CharField(requ ...

  9. [UOJ 74] 【UR #6】破解密码

    题目链接:UOJ - 74 题目分析 题目中,将字符串 S 的第一个字符移到末尾,其他字符向前移动一个位置,f(S) 就从 Hi 变成了 Hi+1. 我们分析一下这个过程:假设第一个字符为 c, (H ...

  10. Servlet处理Cookie

    1.CGI:进程,servlet:线程 2.HttpServletResponse下的方法就没有get开头的,(PrintWriter)getWriter在ServletResponse下. 3.st ...