使用Java Xpath 爬取某易云歌曲

本文使用Java xpath 爬取某易云歌曲，并下载至本地。

代码仅用于个人学习使用，欢迎各位大佬提出建议。

1、添加依赖

        <dependency>
            <groupId>cn.wanghaomiao</groupId>
            <artifactId>JsoupXpath</artifactId>
            <version>2.2</version>
        </dependency>
        <dependency>
            <groupId>cn.hutool</groupId>
            <artifactId>hutool-all</artifactId>
            <version>5.8.9</version>
        </dependency>

2、获取音乐id和url

    /**
     * 获取歌曲信息
     *
     * @param url
     * @return
     */
    public Map<String, Object> getMusicInfo(String url) {
        Map<String, Object> result = new HashMap<>();
        url = url.replace("/#", "");
        Map<String, Object> header = new HashMap<>();
        header.put("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36");
        header.put("Referer", "https://music.163.com/");
        header.put("Host", "music.163.com");
        String res = HttpUtil.get(url, header);

        JXDocument jxDocument = JXDocument.create(res);
        //歌曲列表
        List<JXNode> songs = jxDocument.selN("//ul[@class=\"f-hide\"]/li/a");
        //歌单名称
        JXNode jxsonglistName = jxDocument.selNOne("//h2[contains(@class,\"f-ff2\")]/text()");
        //歌手名
        JXNode jxsingerName = jxDocument.selNOne("//h2[@id=\"artist-name\"]/text()");
        String songlistName = null != jxsonglistName ? jxsonglistName.toString() : "";
        String singerName = null != jxsingerName ? jxsingerName.toString() : "";

        System.out.println(String.format("=======================%s=======================", StrUtil.isBlank(songlistName) ? singerName : songlistName));
        List<Map<String, Object>> musics = new ArrayList<>();
        result.put("title", StrUtil.isBlank(songlistName) ? singerName : songlistName);
        for (JXNode song : songs) {
            Element element = song.asElement();
            String songName = element.text();
            String songId = element.attr("href").split("=")[1];
            String songUrl = OUT_LINK + songId;
            Map<String, Object> map = new HashMap<>();
            map.put("songId", songId);
            map.put("songName", songName);
            map.put("songUrl", songUrl);
            map.put("title", StrUtil.isBlank(songlistName) ? singerName : songlistName);
            //map.put("lyric", getMusicLyric(songId));
            musics.add(map);
            //单线程下载歌曲
            //downloadSong(songName, songUrl, result.get("title").toString());
        }
        musics.forEach(x -> System.out.println(x));

        //多线程下载歌曲
        //multiDownload(musics);
        result.put("songs", musics);
        //System.out.println(result);
        return result;
    }

3、获取歌词

    /**
     * 获取歌词
     *
     * @param songId
     * @return
     */
    public String getMusicLyric(String songId) {
        String url = String.format("http://music.163.com/api/song/lyric?id=%s&lv=-1&kv=-1&tv=-1", songId);
        Map<String, Object> header = new HashMap<>();
        header.put("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36");
        header.put("Referer", "https://music.163.com/");
        header.put("Host", "music.163.com");
        String res = HttpUtil.get(url, header);
        return JSONObject.parseObject(res).getJSONObject("lrc").getString("lyric");
    }

4、完整代码

加入多线程下载歌曲代码

import cn.hutool.core.io.FileUtil;
import cn.hutool.core.lang.Console;
import cn.hutool.core.util.StrUtil;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSONObject;
import org.jsoup.nodes.Element;
import org.seimicrawler.xpath.JXDocument;
import org.seimicrawler.xpath.JXNode;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class Music163 {
    //下载地址
    private static String OUT_LINK = "http://music.163.com/song/media/outer/url?id=";
    //本地下载目录
    private static String DOWNLOAD_PATH = "E:\\music\\";

    public static void main(String[] args) {
        String musicUrl;
        //歌曲清单
        // 热歌 3778678 原创 2884035  新歌 3779629 飙升 19723756
        musicUrl = "https://music.163.com/#/playlist?id=3778678";
        // 歌手歌曲榜单  8325->梁静茹
        //musicUrl = "https://music.163.com/#/artist?id=8325";
        //搜索列表
        // musicUrl = "https://music.163.com/#/search/m/?order=hot&cat=全部&limit=435&offset=435&s=梁静茹";
        Music163 music163 = new Music163();
        music163.getMusicInfo(musicUrl);
    }

    /**
     * 获取歌曲信息
     *
     * @param url
     * @return
     */
    public Map<String, Object> getMusicInfo(String url) {
        Map<String, Object> result = new HashMap<>();
        url = url.replace("/#", "");
        Map<String, Object> header = new HashMap<>();
        header.put("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36");
        header.put("Referer", "https://music.163.com/");
        header.put("Host", "music.163.com");
        String res = HttpUtil.get(url, header);

        JXDocument jxDocument = JXDocument.create(res);
        //歌曲列表
        List<JXNode> songs = jxDocument.selN("//ul[@class=\"f-hide\"]/li/a");
        //歌单名称
        JXNode jxsonglistName = jxDocument.selNOne("//h2[contains(@class,\"f-ff2\")]/text()");
        //歌手名
        JXNode jxsingerName = jxDocument.selNOne("//h2[@id=\"artist-name\"]/text()");
        String songlistName = null != jxsonglistName ? jxsonglistName.toString() : "";
        String singerName = null != jxsingerName ? jxsingerName.toString() : "";

        System.out.println(String.format("=======================%s=======================", StrUtil.isBlank(songlistName) ? singerName : songlistName));
        List<Map<String, Object>> musics = new ArrayList<>();
        result.put("title", StrUtil.isBlank(songlistName) ? singerName : songlistName);
        for (JXNode song : songs) {
            Element element = song.asElement();
            String songName = element.text();
            String songId = element.attr("href").split("=")[1];
            String songUrl = OUT_LINK + songId;
            Map<String, Object> map = new HashMap<>();
            map.put("songId", songId);
            map.put("songName", songName);
            map.put("songUrl", songUrl);
            map.put("title", StrUtil.isBlank(songlistName) ? singerName : songlistName);
            //map.put("lyric", getMusicLyric(songId));
            musics.add(map);
            //单线程下载歌曲
            //downloadSong(songName, songUrl, result.get("title").toString());
        }
        musics.forEach(x -> System.out.println(x));

        //多线程下载歌曲
        //multiDownload(musics);
        result.put("songs", musics);
        //System.out.println(result);
        return result;
    }

    /**
     * 获取歌词
     *
     * @param songId
     * @return
     */
    public String getMusicLyric(String songId) {
        String url = String.format("http://music.163.com/api/song/lyric?id=%s&lv=-1&kv=-1&tv=-1", songId);
        Map<String, Object> header = new HashMap<>();
        header.put("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36");
        header.put("Referer", "https://music.163.com/");
        header.put("Host", "music.163.com");
        String res = HttpUtil.get(url, header);
        return JSONObject.parseObject(res).getJSONObject("lrc").getString("lyric");
    }

    /**
     * 歌曲下载
     *
     * @param songName
     * @param songUrl
     * @param title
     */
    public void downloadSong(String songName, String songUrl, String title) {
        HttpUtil.downloadFile(songUrl,
                FileUtil.file(DOWNLOAD_PATH + title + "\\", songName + ".mp3"));
        Console.log("下载完成！" + "==》" + songName);
    }

    /**
     * 多线程下载
     *
     * @param list
     */
    public void multiDownload(List<Map<String, Object>> list){
        //使用多线程优化查询速度
        int threadNum = 10;
        if (list.size() < 10) {
            threadNum = 1;
        }
        ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
        CountDownLatch countDownLatch = new CountDownLatch(threadNum);

        int perSize = list.size() / threadNum;
        for (int i = 0; i < threadNum; i++) {
            int start = i * perSize;
            int end = (i + 1) * perSize;
            if (i == threadNum - 1) {
                end = list.size();
            }
            List<Map<String, Object>> maps = list.subList(start, end);
            MultiThread thread = new MultiThread();
            thread.setProjectList(maps);
            thread.setCountDownLatch(countDownLatch);
            executorService.submit(thread);
        }
        try {
            countDownLatch.await();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        executorService.shutdown();
    }

    class MultiThread extends Thread {
        private List<Map<String, Object>> projectList;

        private CountDownLatch countDownLatch;

        private List<Map<String, Object>> result;

        public void setResultList(List<Map<String, Object>> result) {
            this.result = result;
        }

        public void setProjectList(List<Map<String, Object>> projectList) {
            this.projectList = projectList;
        }

        public void setCountDownLatch(CountDownLatch countDownLatch) {
            this.countDownLatch = countDownLatch;
        }

        @Override
        public void run() {
            try {

                for (Map<String, Object> map : projectList) {
                    downloadSong((String)map.get("songName"),(String)map.get("songUrl"),(String)map.get("title"));
                }

            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                if (countDownLatch != null) {
                    countDownLatch.countDown();
                }
            }
        }
    }
}