puppeteer爬虫

利用空闲时间，学习了下puppeteer爬虫，我也想爬取下网上的资源

1.部分api

• puppeteer.launch(options)

• browser对象 api

• page对象

2.爬取电影网站

const puppeteer = require('puppeteer');

/* 爬虫的目标链接地址: 豆瓣电影 */
const url = `https://movie.douban.com/tag/#/?sort=R&range=0,10&tags=`;

const sleep = time => new Promise(resolve => {
  setTimeout(resolve, time);
});

(async () => {
  console.log('crawler start to visit the target address');

  /* dumpio 是否将浏览器进程stdout和stderr导入到process.stdout和process.stderr中 */
  const browser = await puppeteer.launch({
    args: ['--no-sandbox'],
    dumpio: false
  });
  const page = await browser.newPage();
  await page.goto(url, {
    waitUntil: 'networkidle2'
  });

  await sleep(3000);

  await page.waitForSelector('.more');

  for(let i = 0; i < 1; i++) {
    await sleep(3000);
    await page.click('.more');
  }

  const result = await page.evaluate(() => {
    let $ = window.$;
    let nodeItems = $('.list-wp a');
    let links = [];
    
    /* 获取对应的元素节点 */
    if(nodeItems.length >= 1) {
      nodeItems.each((index, item) => {
        let elem = $(item);
        let movieId = elem.find('div').data('id');
        let title = elem.find('.title').text();
        let rate = Number(elem.find('.rate').text());
        let poster = elem.find('img').attr('src').replace('s_ratio_poster','l_ratio_poster');

        links.push({
          movieId,
          title,
          rate,
          poster,
        })
      })
    }

    return links;
  });

  browser.close();
  console.log(result)
})();

3.爬取网站内容生成pdf文件

const puppeteer = require('puppeteer');
const url = 'https://cn.vuejs.org/v2/guide/';
(async () => {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();
    await page.goto(url, { waitUntil: 'networkidle0' });
    /* 选择你要输出的那个PDF文件路径，把爬取到的内容输出到PDF中，必须是存在的PDF，可以是空内容，如果不是空的内容PDF，那么会覆盖内容 */
    let pdfFilePath = './index.pdf';
    /* 根据你的配置选项，我们这里选择A4纸的规格输出PDF，方便打印 */
    await page.pdf({
        path: pdfFilePath,
        format: 'A4',
        scale: 1,
        printBackground: true,
        landscape: false,
        displayHeaderFooter: false
    });
    browser.close();
})()

正在努力学习中，若对你的学习有帮助，留下你的印记呗（点个赞咯^_^）