一个简单的puppeteer爬虫

const puppeteer = require("puppeteer");
const path = require('path');
const pathToExtension = path.join(__dirname, './chrome-mac/Chromium.app/Contents/MacOS/Chromium');
const conf = {
    headless: false,
    executablePath: pathToExtension,
    defaultViewport: {
        width: 1300,
        height: 900
    },
};

(async () => {

    const browser = await puppeteer.launch(conf)
    const page = await browser.newPage()

    await page.goto('https://www.baidu.com/', {waitUntil: 'networkidle2'});
    //addScriptTag需要加在goto的后面，然后就可以在evaluate里使用jQuery的语法了。
    await page.addScriptTag({
        url: 'https://code.jquery.com/jquery-3.2.1.min.js',
    });
    await page.waitFor('#u1')
    // 可以接收evaluate内部打印的console内容
    page.on('console',msg=>{
            for(let i =0;i<msg.args().length;i++){
                console.log(`${i}: ${msg.args([i])}`)
            }
    })
     const result = await page.evaluate(() => {
        let data = []; // 初始化空数组来存储数据
        let elements = $("#u1"); // 获取所有元素
        for (let element of elements){
            let title = element.innerText; // 获取标题
            let url = element.href;//获取网址
            data.push({title,url}); // 存入数组
        }
        return data;
    });
    console.log(result);
    await page.waitFor(3000);
    await browser.close();

})();