uniCloud爬虫获取网页数据

'use strict';
let request = require('request')
let cheerio = require('cheerio'); //爬虫
let iconv = require('iconv-lite'); //处理gbk编码的网页
let Entities = require('html-entities').XmlEntities;
let entities = new Entities();
const fs = require('fs')
const path = require('path')
const host = 'http://www.quanshuwang.com/shuku/'

const db = uniCloud.database()
const collection = db.collection('repiles-book')
const dbCom = db.command

exports.main = async (event, context) => {

    // 开始抓取首页链接
    let indexArr = []
    //发送请求获取页面内容
    var body = await requestFn()
    var $ = cheerio.load(body);
    //兼容网页编码格式
    if ($('meta[charset]').attr('charset') == 'utf-8') { //如果网页是utf-8的编码

    } else { //如果网页是gbk的编码
        body = iconv.decode(body, 'gbk'); //转换gbk编码的页面内容
        $ = cheerio.load(body);
    }
    //处理网页数据 获取排行列表
    let list = $('.yd-book-content .tab-item').find('.yd-book-item')
    for (var i = 0; i < list.length; i++) {
        let href = list.eq(i).find('a').attr("href")
        //获取书的id
        let index = href.indexOf('_') + 1
        let index2 = href.lastIndexOf('.')
        let bookId = href.slice(index, index2) //书ID
        //获取书的封面
        let bookImageSrc = list.eq(i).find('img').attr("src")
        //获取书的标题 注意使用html-entities解码
        let bookName = entities.decode(list.eq(i).find('h2').html())
        //获取书的作者 注意使用html-entities解码
        let bookAuthor = entities.decode(list.eq(i).find('.dl-horizontal-inline p').html())

        console.log('书的封面：' + bookImageSrc);
        console.log('书的标题：' + bookName);
        console.log('书的作者：' + bookAuthor);
        console.log('书的id：' + bookId);
        let obj={
            bookImageSrc,
            bookName,
            bookAuthor,
            bookId,
        }
        saveContent(obj)
    }

    console.log('新增文章数量：', indexArr.length);

    // 循环抓取每个新文章详情页
    // if (indexArr.length > 0) {
    //     for (let i = 0; i < indexArr.length; i++) {
    //         let href = list.eq(indexArr[i]).attr("href")
    //         let imgSrc = list.eq(indexArr[i]).find('img').attr('src')
    //         let title = list.eq(indexArr[i]).find('.title').text()
    //         await saveArticle(href, title, imgSrc)
    //     }
    // }

    //返回数据给客户端
    return event
};

function saveContent(obj) {
    //获取要写入文件的文件夹路径（以书名当文件夹）
    let pathBook=path.join(__dirname, `../../bookData/${obj.bookName}`)
    //判断书名文件夹是否存在，不存在则创建
    if (!fs.existsSync(pathBook)) {
        fs.mkdirSync(pathBook)
    }
    //写入json文件
    fs.writeFile(`${pathBook}/chapter1.json`, JSON.stringify(obj), 'utf-8', err => {
        if (err) throw err
    })
}

function requestFn() {
    return new Promise((resolve, reject) => {
        request({
            url: host,
            encoding: null //设置抓取页面时不要对数据做任何转换
        }, function(err, res, body) {
            if (err) {
                reject(err)
            } else {
                resolve(body)
            }
        });
    })
}