Java 爬知乎某个问题下的所有图片

package com.yfzz.zhihu3 ;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.JsonPathSelector;

import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashSet;
import java.util.List;
import java.util.UUID;

public class ZhihuQuestion implements PageProcessor {

    private Site site = Site.me().setSleepTime(2000)
            .setCycleRetryTimes(5)
            .setRetryTimes(5)
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36")
            .addHeader("authorization", "你的认证信息")
            .setCharset("UTF-8");

    private static String questionID = "37787176";  // 要爬取的问题 ID 号
    private static String filePath = "e://zhihu";   // 文件存放路径，程序会自动在此路径后添加一级目录为问题标题
    private static int offset = 0;              // 偏移量，表示从第 n 个答案开始获取，limit 表示获取多少个(上限为20)
    private static int count = 0;               // 下载到的图片总数

    @Override
    public void process(Page page) {
        Html html = page.getHtml();
        // 得到当前页的所有答案的 ID 号，主要用处是为了判断是否到页尾。
        List<String> idList = new JsonPathSelector("$.data[*].id").selectList(page.getRawText());
        String title = new JsonPathSelector("$.data[*].question.title").selectList(page.getRawText()).get(0);

        int getSize = idList.size();        // 当前页获取到的回答个数
        if (getSize == 20) {
            // 将下一页添加到队列中
            offset += 20;
            String url = "https://www.zhihu.com/api/v4/questions/" + questionID + "/answers?include=data%5B%2A%5D.is_normal%2Cadmin_closed_comment%2Creward_info%2Cis_collapsed%2Cannotation_action%2Cannotation_detail%2Ccollapse_reason%2Cis_sticky%2Ccollapsed_by%2Csuggest_edit%2Ccomment_count%2Ccan_comment%2Ccontent%2Ceditable_content%2Cvoteup_count%2Creshipment_settings%2Ccomment_permission%2Ccreated_time%2Cupdated_time%2Creview_info%2Cquestion%2Cexcerpt%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%2Cupvoted_followees%3Bdata%5B%2A%5D.mark_infos%5B%2A%5D.url%3Bdata%5B%2A%5D.author.follower_count%2Cbadge%5B%3F%28type%3Dbest_answerer%29%5D.topics&offset=" + offset + "&limit=20&sort_by=default";
            page.addTargetRequest(url);
        }

        // 根据正则匹配图片地址并去除特殊字符。
        // 例: 正则得到的图片链接 : \"https://pic2.zhimg.com/v2-94ae015b8e1bd2e0dc9bdd7d7da6d7ed_r.jpg\" 需要去除特殊字符 "
        List<String> imgList = html.regex("data-original=\"(.*?)\"").replace("\\\\"", "").all();

        // 图片链接去重复
        HashSet<String> set = new HashSet<String>(imgList);
        count += set.size();
        
        System.out.println("正在下载第" + offset + "-" + (offset + getSize) + "个回答的图片，当前页图片数量为：" + set.size()  + ",目前总图片数量：" + count);
        for (String url : set) {
            String fileName = url.substring(url.lastIndexOf('/') + 1, url.length());
            try {
                String savePath = filePath + "/" + title + "_" + questionID;    // 下载路径：指定路径/标题/问题ID
                downloadPicture(url, savePath, fileName);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }


    public static void downloadPicture(String urlString, String savePath, String filename) throws Exception {
        File file = new File(savePath + File.separator + filename);

        if (!new File(savePath).exists()) {
            System.out.println("下载目录不存在，已创建：" + savePath);
            new File(savePath).mkdirs();
        }
        if (file.exists()) {
            System.out.println("文件已存在，跳过该文件：" + file.getName());
            return;
        }

        OutputStream os = new FileOutputStream(file);
        // 构造URL
        URL url = new URL(urlString);
        // 打开连接
        URLConnection con = url.openConnection();
        //设置请求超时为5s
        con.setConnectTimeout(5 * 1000);
        // 输入流
        InputStream is = con.getInputStream();

        // 1K的数据缓冲
        byte[] bs = new byte[1024];
        // 读取到的数据长度
        int len;
        // 开始读取
        while ((len = is.read(bs)) != -1) {
            os.write(bs, 0, len);
        }
        // 完毕，关闭所有链接
        os.close();
        is.close();
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) throws Exception {
        Spider.create(new ZhihuQuestion()).
                thread(1).
                addUrl("https://www.zhihu.com/api/v4/questions/" + questionID + "/answers?include=data%5B%2A%5D.is_normal%2Cadmin_closed_comment%2Creward_info%2Cis_collapsed%2Cannotation_action%2Cannotation_detail%2Ccollapse_reason%2Cis_sticky%2Ccollapsed_by%2Csuggest_edit%2Ccomment_count%2Ccan_comment%2Ccontent%2Ceditable_content%2Cvoteup_count%2Creshipment_settings%2Ccomment_permission%2Ccreated_time%2Cupdated_time%2Creview_info%2Cquestion%2Cexcerpt%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%2Cupvoted_followees%3Bdata%5B%2A%5D.mark_infos%5B%2A%5D.url%3Bdata%5B%2A%5D.author.follower_count%2Cbadge%5B%3F%28type%3Dbest_answerer%29%5D.topics&offset=0&limit=20&sort_by=default")
                .run();
    }
}