go post 参数_go语言实现网易云音乐爬虫

前言

最近在学习go，学习一门语言最好的方式就是实践，之前学习python也是从爬虫入手，现在使用go语言写一个网易云音乐的爬虫，下面会简单介绍开发的过程，代码是初学者的水平，欢迎吐槽。

本项目github地址https://github.com/zhujiajunup/yunyinyue

开发工具

- go1.11.2 windows/amd64

- Google Chrome 71.0.3578.98

- Fiddler v5.0.20182.28034

获取数据

不管用什么语言写爬虫，但步骤总是一致的，只是实现使用不应的语言而已，第一步当然是确认你想要什么，本次的目标是网易云音乐，我是想获取用户首页的听歌排行榜。

最先需要弄明白的是这些数据是怎么获取的，即云音乐是如何向服务器请求数据的，打开chrome的调试工具（F12）,点到Network，搜索“钟无艳”

可以看到数据是https://music.163.com/weapi/v1/play/record?csrf_token=的POST请求来获取的，再看看该请求发送了什么数据

可以看到提交了一个表单，参数为params和encSecKey。发送的数据应该是加了密的，下一步就需要知道云音乐是如何进行加密传输的。

从调试窗口可以看到，该请求是由https://s3.music.126.net/web/s/core_86994123ce247287ad52aafce6acdf9b.js?86994123ce247287ad52aafce6acdf9b发出的，加密逻辑应该就是在该js中处理的，将该js保存到本地并格式化后，并搜索encSecKey在哪里赋值

v9m.bl9c = function(Y9P, e8e) {var i8a = {},e8e = NEJ.X({},e8e),mp3x = Y9P.indexOf("?");if (window.GEnc && /(^|.com)/api/.test(Y9P) && !(e8e.headers && e8e.headers[eq1x.Bx8p] == eq1x.Iy0x) && !e8e.noEnc) {if (mp3x != -1) {i8a = k8c.hc2x(Y9P.substring(mp3x + 1));Y9P = Y9P.substring(0, mp3x)}if (e8e.query) {i8a = NEJ.X(i8a, k8c.fQ1x(e8e.query) ? k8c.hc2x(e8e.query) : e8e.query)}if (e8e.data) {i8a = NEJ.X(i8a, k8c.fQ1x(e8e.data) ? k8c.hc2x(e8e.data) : e8e.data)}i8a["csrf_token"] = v9m.gO2x("__csrf");Y9P = Y9P.replace("api", "weapi");e8e.method = "post";delete e8e.query;var bUK2x = window.asrsea(JSON.stringify(i8a), brA4E(["流泪", "强"]), brA4E(WU5Z.md), brA4E(["爱心", "女孩", "惊恐", "大笑"]));e8e.data = k8c.cz9q({params: bUK2x.encText,encSecKey: bUK2x.encSecKey})}cwC9t(Y9P, e8e)};

可以看到是通过window.asrsea函数来获取的，window.asrsea的定义如下：

function() {function a(a) {var d, e, b = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789",c = "";for (d = 0; a > d; d += 1) e = Math.random() * b.length,e = Math.floor(e),c += b.charAt(e);return c}function b(a, b) {var c = CryptoJS.enc.Utf8.parse(b),d = CryptoJS.enc.Utf8.parse("0102030405060708"),e = CryptoJS.enc.Utf8.parse(a),f = CryptoJS.AES.encrypt(e, c, {iv: d,mode: CryptoJS.mode.CBC});return f.toString()}function c(a, b, c) {var d, e;return setMaxDigits(131),d = new RSAKeyPair(b, "", c),e = encryptedString(d, a)}function d(d, e, f, g) {var h = {},i = a(16);return h.encText = b(d, g),h.encText = b(h.encText, i),h.encSecKey = c(i, e, f),h}function e(a, b, d, e) {var f = {};return f.encText = c(a + e, b, d),f}window.asrsea = d,window.ecnonasr = e
} ();

加密算法就是这段代码，函数接收四个参数，进行了aes和ras加密，具体逻辑这里不进行详解，知道了处理逻辑，现在得获取这四个参数分别是什么，接下来使用Fiddler来将js替换为本地js,将参数打印出来即可。

Fiddler调试获取参数

配置代理

配置Https

修改core.js 将https://s3.music.126.net/web/s/core_86994123ce247287ad52aafce6acdf9b.js?86994123ce247287ad52aafce6acdf9b保存到本地，比如命名为core.js，编辑器打开编辑，在指定位置添加如下打印信息

v9m.bl9c = function(Y9P, e8e) {var i8a = {},e8e = NEJ.X({},e8e),mp3x = Y9P.indexOf("?");if (window.GEnc && /(^|.com)/api/.test(Y9P) && !(e8e.headers && e8e.headers[eq1x.Bx8p] == eq1x.Iy0x) && !e8e.noEnc) {if (mp3x != -1) {i8a = k8c.hc2x(Y9P.substring(mp3x + 1));Y9P = Y9P.substring(0, mp3x)}if (e8e.query) {i8a = NEJ.X(i8a, k8c.fQ1x(e8e.query) ? k8c.hc2x(e8e.query) : e8e.query)}if (e8e.data) {i8a = NEJ.X(i8a, k8c.fQ1x(e8e.data) ? k8c.hc2x(e8e.data) : e8e.data)}i8a["csrf_token"] = v9m.gO2x("__csrf");Y9P = Y9P.replace("api", "weapi");e8e.method = "post";delete e8e.query;var bUK2x = window.asrsea(JSON.stringify(i8a), brA4E(["流泪", "强"]), brA4E(WU5Z.md), brA4E(["爱心", "女孩", "惊恐", "大笑"]));window.console.info(Y9P);window.console.info(JSON.stringify(i8a));window.console.info(JSON.stringify( brA4E(["流泪", "强"])));window.console.info(JSON.stringify(brA4E(WU5Z.md)));window.console.info(JSON.stringify(brA4E(["爱心", "女孩", "惊恐", "大笑"])));e8e.data = k8c.cz9q({params: bUK2x.encText,encSecKey: bUK2x.encSecKey})}cwC9t(Y9P, e8e)};

配置Fiddler Rule

一切准备就绪后，再打开浏览器，输入https://music.163.com/#/user/songs/rank?id=62947535，打开调试窗口得Console，就可以看到本地js中添加的输出日志了

不难发现除了第一个参数外，其他三个参数都是固定的，因此在后面的处理中，只需要处理第一个参数即可。

第一个参数就是加密前的请求参数

{"uid": "62947535","type": "-1","limit": "1000","offset": "0","total": "true","csrf_token": ""
}

那么已经弄清楚了数据获取的逻辑，接下来就是按照这个逻辑用go语言实现一遍了，最关键的就是加密算法了

go实现

项目结构

加密算法

来自https://studygolang.com/topics/5815

/*
Package encrypt provides encrypt algorithm such as rsa & aes
*/
package encryptimport ("bytes""crypto/aes""crypto/cipher""encoding/base64""fmt""math/big""math/rand""time"
)// generate string for given size
func RandomStr(size int) (result []byte) {s := "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"strBytes := []byte(s)r := rand.New(rand.NewSource(time.Now().UnixNano()))for i := 0; i < size; i++ {result = append(result, strBytes[r.Intn(len(strBytes))])}return
}func AesEncrypt(sSrc string, sKey string, aseKey string) (string, error) {iv := []byte(aseKey)block, err := aes.NewCipher([]byte(sKey))if err != nil {return "", err}padding := block.BlockSize() - len([]byte(sSrc))%block.BlockSize()src := append([]byte(sSrc), bytes.Repeat([]byte{byte(padding)}, padding)...)model := cipher.NewCBCEncrypter(block, iv)cipherText := make([]byte, len(src))model.CryptBlocks(cipherText, src)return base64.StdEncoding.EncodeToString(cipherText), nil
}func RsaEncrypt(key string, pubKey string, modulus string) string {rKey := ""for i := len(key) - 1; i >= 0; i-- { // reserve keyrKey += key[i : i+1]}hexRKey := ""for _, char := range []rune(rKey) {hexRKey += fmt.Sprintf("%x", int(char))}bigRKey, _ := big.NewInt(0).SetString(hexRKey, 16)bigPubKey, _ := big.NewInt(0).SetString(pubKey, 16)bigModulus, _ := big.NewInt(0).SetString(modulus, 16)bigRs := bigRKey.Exp(bigRKey, bigPubKey, bigModulus)hexRs := fmt.Sprintf("%x", bigRs)return addPadding(hexRs, modulus)
}func addPadding(encText string, modulus string) string {ml := len(modulus)for i := 0; ml > 0 && modulus[i:i+1] == "0"; i++ {ml--}num := ml - len(encText)prefix := ""for i := 0; i < num; i++ {prefix += "0"}return prefix + encText
}

Music163Spider

type Music163Spider struct {// send requestclient  *http.Client// request's headerheaders map[string]string
}func NewMusic164Spider() (spider Music163Spider) {headers := make(map[string]string)headers["Accept"] = "ext/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"// empty hereheaders["Accept-Encoding"] = ""headers["Content-Type"] = "application/x-www-form-urlencoded"headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"headers["Host"] = constants.Music163Hostheaders["Cache-Control"] = "no-cache"headers["Connection"] = "keep-alive"headers["Pragma"] = "no-cache"headers["Origin"] = fmt.Sprintf("%s%s", constants.HttpsPrefix, constants.Music163Host)headers["Accept"] = "ext/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"return Music163Spider{client:  &http.Client{},headers: headers,}
}

加密函数，实现window.asrsea的加密功能

func (spider Music163Spider) dataEncrypt(dataBytes []byte) (content map[string]string) {content = make(map[string]string)randomBytes := encrypt.RandomStr(16)params, err := encrypt.AesEncrypt(string(dataBytes), constants.SrcretKey, constants.AseKey)if err != nil {fmt.Println(err)}params, err = encrypt.AesEncrypt(params, string(randomBytes), constants.AseKey)if err != nil {fmt.Println(err)}encSecKey := encrypt.RsaEncrypt(string(randomBytes), constants.PubKey, constants.Modulus)if err != nil {fmt.Println(err)}content["params"] = string(params)content["encSecKey"] = string(encSecKey)return content
}

定义了发送post请求的方法

func (spider Music163Spider) httpPost(url string, headers map[string]string, params interface{}) (result []byte, err error) {body := make(url2.Values)jsonParams, err := json.Marshal(params)if err != nil {return nil, err}encryptResultMap := spider.dataEncrypt(jsonParams)body["params"] = []string{encryptResultMap["params"]}body["encSecKey"] = []string{encryptResultMap["encSecKey"]}req, err := http.NewRequest("POST", url, strings.NewReader(body.Encode()))for key, value := range headers {req.Header.Add(key, value)}if err != nil {return nil, err}resp, err := spider.client.Do(req)defer resp.Body.Close()data, err := ioutil.ReadAll(resp.Body)if err != nil {return nil, err}return data, nil
}

发送参数

type BaseRequestBody struct {Offset    string `json:"offset"`Total     string `json:"totail"`Limit     string `json:"limit"`CsrfToken string `json:"csrf_token"`
}type PlayRecordRequestBody struct {BaseRequestBodyType string `json:"type"`Uid  string `json:"uid"`
}

根据获取排行榜请求返回数据来创建相应的对象

type SongDetail struct {Song common.Song `json:"song"`// ignore other
}
type PlayRecord struct {PlayCount  int        `json:"playCount"`Score      int        `json:"score"`SongDetail SongDetail `json:"song"`
}type PlayRecordResp struct {Code     int          `json:"code"`AllData  []PlayRecord `json:"allData"`WeekData []PlayRecord `json:"weekData"`
}

定义好对象后，就可以使用模拟发送请求了, spider的GetPlayRecord方法

func (spider Music163Spider) GetPlayRecord(userId string) (record response.PlayRecordResp, err error) {playRecordReqBody := request.PlayRecordRequestBody{Uid:  userId,Type: "-1",BaseRequestBody: request.BaseRequestBody{Offset:    "0",Total:     "true",Limit:     "1000",CsrfToken: "",},}playRecordUrl := fmt.Sprintf("%s%s%s?csrf_token=", constants.HttpsPrefix, constants.Music163Host, constants.PlayRecord)result, err := spider.httpPost(playRecordUrl, spider.headers, playRecordReqBody)if err != nil {return}playRecordResp := response.PlayRecordResp{}json.Unmarshal([]byte(result), &playRecordResp)return playRecordResp, nil
}

测试一下

func main() {musicSpider := spider.NewMusic164Spider()record, _ := musicSpider.GetPlayRecord("62947535")jsonData, _ := json.MarshalIndent(record, "", "t")fmt.Println(string(jsonData))
}

结果输出

Reference

Go in action
golang +es 爬取网易云音乐评论(整理版本1)
Python 从零开始爬虫(七)——实战：网易云音乐评论爬取（附加密算法）go实现网易云音乐爬虫go实现网易云音乐爬虫