基于steam的游戏销量预测 — PART 1 — 爬取steam游戏相关数据的爬虫

语言：python

环境：ubuntu

爬取内容：steam游戏标签，评论，以及在 steamspy 爬取对应游戏的销量

使用相关：urllib，lxml，selenium，chrome

解释：

　　流程图如下

　　1.首先通过 steam 商店搜索页面的链接，打开 steam 搜索页面，然后用如下正则表达式来得到前100个左右的游戏的商店页面链接。

reg = r'<a href="(http://store.steampowered.com/app/.+?)"'

　　2.对于得到的每个商店页面链接，可以通过如下正则表达式来得到对应的有游戏名称.

reg = r'.+?/app/[0-9]+?/(.+?)/'

　　　例如如下链接 http://store.steampowered.com/app/268910/Cuphead/ ，可以得到游戏名字为Cuphead。

　　3.然后通过 selenium 来模拟 chrome 上的操作，以获取动态加载的网页。先打开网页 steamspy，然后在网页上检查元素，看源码，发现搜索框元素的 name 值为”s”，所以可以通过 driver.find_element_by_name("s") 找到搜索框，模拟输入对应的游戏名字。进行搜索，得到了新的页面，再通过如下正则表达式得到销量

reg = r'<strong>Owners</strong>:\s+?([0-9,]+?)\s+?'

　　　例如上面那个网址对应应当输入 Cuphead。

　　4.得到游戏标签，这一步比较简单，打开商店链接，得到源码，然后通过如下正则表达式获取标签即可

reg=r'>\s+?([^\t]+?)\s+?</a><a href="http://store.steampowered.com/tag.+?"\s+?class="app_tag"'

　　5.得到游戏评论。由于 steam 商店评论是动态加载的，所以要又通过 selenium 来模拟 chrome 的操作，首先进入商店页面，因为有些商店是有年龄确认的按钮存在，那么通过 xpath 来找 viewpage 的按钮，如果有按钮则模拟点击操作，否则不点击。代码如下

driver.find_element_by_xpath("//span[text()='View Page']").click()

　　6.这样就进入了商店页面，然后类似地，通过xpath找到加载评论的按钮，加载评论，代码如下。

driver.find_element_by_xpath("//span[starts-with(@class,'game_review_summary')]").click()

　　7.再通过xpath找到多条评论的链接，代码如下。

elements = driver.find_elements_by_xpath("//a[starts-with(@href,'http://steamcommunity.com/id')]")

　　8.得到评论链接之后，打开评论链接，并通过如下正则表达式来得到评论正文内容。

reg = r'<div\s+?id="ReviewText">(.+?)</div>'

代码：

 import urllib
 import re
 import sys
 import lxml
 from selenium import webdriver
 from selenium.webdriver.common.keys import Keys

 def getHtml(url):
     page = urllib.urlopen(url)
     html = page.read()
     return html

 def getGameLink(html):
     reg = r'<a href="(http://store.steampowered.com/app/.+?)"'
     gamelinkre = re.compile(reg)
     gamelinklist = re.findall(gamelinkre,html)
     return gamelinklist

 def getTag(html):
     reg = r'>\s+?([^\t]+?)\s+?</a><a href="http://store.steampowered.com/tag.+?"\s+?class="app_tag"'
     tagre = re.compile(reg)
     taglist = re.findall(tagre,html)
     return taglist

 def getReviewLink(url):
     gamereviewlinklist = []
     driver = webdriver.Chrome()
     flag = True
     try:
         driver.get(url)
         driver.implicitly_wait(30)
         flag = True
     except:
         return gamereviewlinklist
     try:
         driver.find_element_by_xpath("//span[text()='View Page']").click()
         driver.implicitly_wait(30)
         flag = True
     except:
         flag = False
     try:
         driver.find_element_by_xpath("//span[starts-with(@class,'game_review_summary')]").click()
         driver.implicitly_wait(30)
         flag = True
     except:
         flag = False
     if(flag == False):
         driver.quit()
         return gamereviewlinklist
     elements = driver.find_elements_by_xpath("//a[starts-with(@href,'http://steamcommunity.com/id')]")
     pattern = re.compile(r'recommended/.+')
     for element in elements:
         url = element.get_attribute("href")
         if(re.search(pattern,url)):
            gamereviewlinklist.append(url)
     driver.quit()
     return gamereviewlinklist

 def getReview(html):
     reg = r'<div\s+?id="ReviewText">(.+?)</div>'
     reviewre = re.compile(reg)
     reviewlist = re.findall(reviewre,html)
     reviewlist.append("")
     print reviewlist[0]
     return reviewlist[0]

 def getSale(url):
     searchwebname="http://steamspy.com/search.php"
     reg = r'.+?/app/[0-9]+?/(.+?)/'
     namere = re.compile(reg)
     nameresult = re.findall(namere,url)
     name = nameresult[0]
     print name
     driver = webdriver.Chrome()
     driver.get(searchwebname)
     driver.implicitly_wait(30)
     flag = True
     elem = driver.find_element_by_name("s")
     elem.clear()
     elem.send_keys(name)
     driver.implicitly_wait(30)
     elem.send_keys(Keys.RETURN)
     driver.implicitly_wait(30)
     pagesource = driver.page_source
     reg = r'<strong>Owners</strong>:\s+?([0-9,]+?)\s+?'
     salere = re.compile(reg)
     saleresult = re.findall(salere,pagesource)
     sale = "-1"
     if len(saleresult)>0:
         sale = saleresult[0]
     print sale
     driver.quit()
     return sale

 reload(sys)
 sys.setdefaultencoding('utf-8')

 urls = []
 inputfilename = "urls.txt"
 inputfile = file(inputfilename,'r')
 emptyflag = 0
 while not emptyflag:
     nowline = inputfile.readline()
     if(nowline == ""):
         emptyflag = 1
     else:
         urls.append(nowline)
 inputfile.close()

 gamelinklist = []
 for urli in urls:
     html = getHtml(urli)
     gamelinklist.extend(getGameLink(html))

 salefilename = "gamesales.txt"
 salefile = file(salefilename,"w")
 for gamelinki in gamelinklist:
     sale = getSale(gamelinki)
     print sale
     print >> salefile,gamelinki
     print >> salefile,sale
     print >> salefile,"sale end"
     print gamelinki+"--sale end"
 salefile.close()

 tagfilename = "gametags.txt"
 tagfile = file(tagfilename,"w")
 for gamelinki in gamelinklist:
     html = getHtml(gamelinki)
     taglist = getTag(html)
     print taglist
     print >> tagfile,gamelinki
     for tagi in taglist:
         print >> tagfile,tagi
     print >> tagfile,"tag end"
     print gamelinki+"--tag end"
 tagfile.close()

 reviewfilename = "gamereviews.txt"
 reviewfile = file(reviewfilename,"w")
 lst = ""
 for gamelinki in gamelinklist:
     reviewlinklist = getReviewLink(gamelinki)
     print reviewlinklist
     print >> reviewfile,gamelinki
     for reviewlinki in reviewlinklist:
         if(reviewlinki != lst):
             html = getHtml(reviewlinki)
             review = getReview(html)
             print >> reviewfile,review
             print >> reviewfile,"a review end"
             lst = reviewlinki
     print >> reviewfile,"review end"
     print gamelinki+"--review end"
 reviewfile.close()