京东口红top 30分析

一、抓取商品id

分析网页源码，发现所有id都是在class=“gl-item”的标签里，可以利用bs4的select方法查找标签，获取id：

获取id后，分析商品页面可知道每个商品页面就是id号不同，可构造url：

将获取的id和构造的url保存在列表里，如下源码：

 def get_product_url(url):
     global pid
     global links
     req = urllib.request.Request(url)
     req.add_header("User-Agent",
                    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
                    '(KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36')
     req.add_header("GET", url)
     content = urllib.request.urlopen(req).read()
     soup = bs4.BeautifulSoup(content, "lxml")
     product_id = soup.select('.gl-item')
     for i in range(len(product_id)):
         lin = "https://item.jd.com/" + str(product_id[i].get('data-sku')) + ".html"
         # 获取链接
         links.append(lin)
         # 获取id
         pid.append(product_id[i].get('data-sku'))

二、获取商品信息

通过商品页面获取商品的基本信息（商品名，店名，价格等）：

         product_url = links[i]
         req = urllib.request.Request(product_url)
         req.add_header("User-Agent",
                        'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0')
         req.add_header("GET", product_url)
         content = urllib.request.urlopen(req).read()
         # 获取商品页面源码
         soup = bs4.BeautifulSoup(content, "lxml")
         # 获取商品名
         sku_name = soup.select_one('.sku-name').getText().strip()
         # 获取商店名
         try:
             shop_name = soup.find(clstag="shangpin|keycount|product|dianpuname1").get('title')
         except:
             shop_name = soup.find(clstag="shangpin|keycount|product|zcdpmc_oversea").get('title')
         # 获取商品ID
         sku_id = str(pid[i]).ljust(20)
         # 获取商品价格

通过抓取评论的json页面获取商品热评、好评率、评论：

获取热评源码：

 def get_product_comment(product_id):
     comment_url = 'https://club.jd.com/comment/productPageComments.action?' \
                   'callback=fetchJSON_comment98vv16496&' \
                   'productId={}&' \
                   'score=0&' \
                   'sortType=6&' \
                   'page=0&' \
                   'pageSize=10' \
                   '&isShadowSku=0'.format(str(product_id))
     response = urllib.request.urlopen(comment_url).read().decode('gbk', 'ignore')
     response = re.search(r'(?<=fetchJSON_comment98vv16496\().*(?=\);)', response).group(0)
     response_json = json.loads(response)
     # 获取商品热评
     hot_comments = []
     hot_comment = response_json['hotCommentTagStatistics']
     for h_comment in hot_comment:
         hot = str(h_comment['name'])
         count = str(h_comment['count'])
         hot_comments.append(hot + '(' + count + ')')
     return ','.join(hot_comments)

获取好评率源码：

 def get_good_percent(product_id):
     comment_url = 'https://club.jd.com/comment/productPageComments.action?' \
                   'callback=fetchJSON_comment98vv16496&' \
                   'productId={}&' \
                   'score=0&' \
                   'sortType=6&' \
                   'page=0&' \
                   'pageSize=10' \
                   '&isShadowSku=0'.format(str(product_id))
     response = requests.get(comment_url).text
     response = re.search(r'(?<=fetchJSON_comment98vv16496\().*(?=\);)', response).group(0)
     response_json = json.loads(response)
     # 获取好评率
     percent = response_json['productCommentSummary']['goodRateShow']
     percent = str(percent) + '%'
     return percent

获取评论源码：

 def get_comment(product_id, page):
     global word
     comment_url = 'https://club.jd.com/comment/productPageComments.action?' \
                   'callback=fetchJSON_comment98vv16496&' \
                   'productId={}&' \
                   'score=0&' \
                   'sortType=6&' \
                   'page={}&' \
                   'pageSize=10' \
                   '&isShadowSku=0'.format(str(product_id), str(page))
     response = urllib.request.urlopen(comment_url).read().decode('gbk', 'ignore')
     response = re.search(r'(?<=fetchJSON_comment98vv16496\().*(?=\);)', response).group(0)
     response_json = json.loads(response)
     # 写入评论.csv
     comment_file = open('{0}\\评论.csv'.format(path), 'a', newline='', encoding='utf-8', errors='ignore')
     write = csv.writer(comment_file)
     # 获取用户评论
     comment_summary = response_json['comments']
     for content in comment_summary:
         # 评论时间
         creation_time = str(content['creationTime'])
         # 商品颜色
         product_color = str(content['productColor'])
         # 商品名称
         reference_name = str(content['referenceName'])
         # 客户评分
         score = str(content['score'])
         # 客户评论
         content = str(content['content']).strip()
         # 记录评论
         word.append(content)
         write.writerow([product_id, reference_name, product_color, creation_time, score, content])
     comment_file.close()

整体获取商品信息源码：

 def get_product_info():
     global pid
     global links
     global word
     # 创建评论.csv
     comment_file = open('{0}\\评论.csv'.format(path), 'w', newline='')
     write = csv.writer(comment_file)
     write.writerow(['商品id', '商品', '颜色', '评论时间', '客户评分', '客户评论'])
     comment_file.close()
     # 创建商品.csv
     product_file = open('{0}\\商品.csv'.format(path), 'w', newline='')
     product_write = csv.writer(product_file)
     product_write.writerow(['商品id', '所属商店', '商品', '价格', '商品好评率', '商品评价'])
     product_file.close()

     for i in range(len(pid)):
         print('[*]正在收集数据。。。')
         product_url = links[i]
         req = urllib.request.Request(product_url)
         req.add_header("User-Agent",
                        'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0')
         req.add_header("GET", product_url)
         content = urllib.request.urlopen(req).read()
         # 获取商品页面源码
         soup = bs4.BeautifulSoup(content, "lxml")
         # 获取商品名
         sku_name = soup.select_one('.sku-name').getText().strip()
         # 获取商店名
         try:
             shop_name = soup.find(clstag="shangpin|keycount|product|dianpuname1").get('title')
         except:
             shop_name = soup.find(clstag="shangpin|keycount|product|zcdpmc_oversea").get('title')
         # 获取商品ID
         sku_id = str(pid[i]).ljust(20)
         # 获取商品价格
         price_url = 'https://p.3.cn/prices/mgets?pduid=1580197051&skuIds=J_{}'.format(pid[i])
         response = requests.get(price_url).content
         price = json.loads(response)
         price = price[0]['p']
         # 写入商品.csv
         product_file = open('{0}\\商品.csv'.format(path), 'a', newline='', encoding='utf-8', errors='ignore')
         product_write = csv.writer(product_file)
         product_write.writerow(
             [sku_id, shop_name, sku_name, price, get_good_percent(pid[i]), get_product_comment(pid[i])])
         product_file.close()
         pages = int(get_comment_count(pid[i]))
         word = []
         try:
             for j in range(pages):
                 get_comment(pid[i], j)
         except Exception as e:
             print("[!!!]{}商品评论加载失败！".format(pid[i]))
             print("[!!!]Error：{}".format(e))

         print('[*]第{}件商品{}收集完毕！'.format(i + 1, pid[i]))         # 的生成词云
         word = " ".join(word)
         my_wordcloud = WordCloud(font_path='C:\Windows\Fonts\STZHONGS.TTF', background_color='white').generate(word)
         my_wordcloud.to_file("{}.jpg".format(pid[i]))

将商品信息和评论写入表格，生成评论词云：

三、总结

在爬取的过程中遇到最多的问题就是编码问题，获取页面的内容requset到的都是bytes类型的要decode（”gbk”），后来还是存在编码问题，最后找到一些文章说明，在后面加“ignore”可以解决，由于爬取的量太大，会有一些数据丢失，不过数据量够大也不影响对商品分析。