package pholcus_lib// 基础包
import (// "github.com/henrylee2cn/pholcus/common/goquery"                          //DOM解析"github.com/henrylee2cn/pholcus/app/downloader/request" //必需. "github.com/henrylee2cn/pholcus/app/spider"           //必需// . "github.com/henrylee2cn/pholcus/app/spider/common" //选用// "github.com/henrylee2cn/pholcus/logs"// net包// "net/http" //设置http.Header// "net/url"// 编码包// "encoding/xml"//"encoding/json"// 字符串处理包//"regexp"// "strconv"// "fmt"// "math"//"net/http""strconv""regexp""fmt""encoding/json""net/url"//"strings"//"strings""strings""github.com/henrylee2cn/pholcus/common/goquery"//"net/http"
)
获取unix时间
var millisecond int64
//用户名
var name string
//密码
var password string
//解析json结构体
type (Info struct {Retcode  intUid stringNick stringCrossDomainUrlList []string}
)
func init() {FileTest.Register()millisecond = getMillisecond()name="88888888"password="8888888"name = encryptUname(name)}var FileTest = &Spider{Name:        "微博登录测试",Description: "测试 [s.weibo.com/user/]",Pausetime: 1500,Keyin:   KEYIN,// Limit:        LIMIT,EnableCookie: true,RuleTree: &RuleTree{Root: func(ctx *Context) {//https://weibo.cn/ctx.AddQueue(&request.Request{Url:          "https://login.sina.com.cn/sso/prelogin.php?entry=account&callback=sinaSSOController.preloginCallBack&su="+name+"&rsakt=mod&client=ssologin.js(v1.4.15)&_="+strconv.FormatInt(millisecond,10),Rule:         "登录一",//DownloaderID:1,})},Trunk: map[string]*Rule{"登录一": {ParseFunc: func(ctx *Context) {str := ctx.GetText()println("-----1-----" + str)compile1, _ := regexp.Compile("{.*}")match1 := compile1.FindString(str)fmt.Println(match1)//json str 转mapvar dat map[string]interface{}if err := json.Unmarshal([]byte(match1), &dat); err == nil {if err != nil{println("转换异常！")}}servertime := dat["servertime"]servertime= strconv.FormatFloat(servertime.(float64), 'f', -1, 64)nonce:=dat["nonce"]pubkey:=dat["pubkey"]rsakv := dat["rsakv"]//加密密码ep := encryptPassword(pubkey.(string), servertime.(string), nonce.(string), password)postDict := map[string]string{}postDict["entry"] = "account"postDict["gateway"] = "1"postDict["from"] = ""postDict["savestate"] = "30"postDict["qrcode_flag"] = "true"postDict["useticket"] = "0"postDict["pagerefer"] = ""postDict["vsnf"] = "1"postDict["su"] = namepostDict["service"] = "account"postDict["servertime"] = servertime.(string)postDict["nonce"] = nonce.(string)postDict["pwencode"] = "rsa2"postDict["rsakv"] = rsakv.(string)postDict["sp"] = eppostDict["sr"] = "1395*822"postDict["cdult"] = "3"postDict["domain"] = "sina.com.cn"postDict["prelt"] = "170"postDict["returntype"] = "TEXT"postValues := url.Values{}for postKey, PostValue := range postDict{postValues.Set(postKey, PostValue)}//post参数编码postDataStr := postValues.Encode()ctx.AddQueue(&request.Request{Url:          "https://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.15)&_="+strconv.FormatInt(getMillisecond(),10),Method:       "POST",EnableCookie: true,PostData:     postDataStr,Rule:         "登录二",//DownloaderID:1,})},},"登录二": {ParseFunc: func(ctx *Context) {str := ctx.GetText()println("-----2-----" + str)var dat Infojson.Unmarshal([]byte(str), &dat)//此处获取2个链接，包含普通版和移动版//print(dat.CrossDomainUrlList[2])ctx.AddQueue(&request.Request{Url:          dat.CrossDomainUrlList[2],Method:       "GET",EnableCookie: true,Rule:         "登录三",})},},"登录三": {ParseFunc: func(ctx *Context) {ctx.AddQueue(&request.Request{Url:          "https://weibo.cn/",Method:       "GET",EnableCookie: true,Rule:         "重定向一",})},},"重定向一": {ParseFunc: func(ctx *Context) {compile2, _ := regexp.Compile("[a-zA-z]+://[^\\s]*")string := compile2.FindAllString(ctx.GetText(), 2)ctx.AddQueue(&request.Request{Url:          string[1],Method:       "GET",EnableCookie: true,Rule:         "重定向二",})},},"重定向二": {ParseFunc: func(ctx *Context) {compile2, _ := regexp.Compile("[a-zA-z]+://[^\\s]*")string := compile2.FindAllString(ctx.GetText(), 3)ctx.AddQueue(&request.Request{Url:          string[2],Method:       "GET",EnableCookie: true,Rule:         "进入首页",})},},"进入首页": {ParseFunc: func(ctx *Context) {for z := 1;z<=2;z++{ctx.AddQueue(&request.Request{Url:          "https://weibo.cn/search/user/?keyword="+ ctx.GetKeyin() + "&page=" + strconv.Itoa(z),// //,Rule:         "查找微博",Method:       "GET",EnableCookie: true,//PostData:"keyword=财经&suser=找人",//DownloaderID:1,smblog})}},},"查找微博": {ParseFunc: func(ctx *Context) {println("---------------查找微博-------------")query := ctx.GetDom()navBox := query.Find("table")navBox.Each(func(i int, s *goquery.Selection) {str := s.Find("tr").Text()j := strings.LastIndex(str,"粉丝")z := strings.LastIndex(str,"人")//昵称name := str[0:j]//粉丝数fansNum := str[j+6:z]//地区city := str[z+5:len(str)]println("name" + name)//链接if url, ok := s.Find("table tr td a").Attr("href"); ok {ctx.AddQueue(&request.Request{Url:  "https://weibo.cn" + url,Rule: "博主首页",Temp: map[string]interface{}{"name":  name,"fansNum": fansNum,"city":  city,},})}})},},"博主首页": {ParseFunc: func(ctx *Context) {//昵称name := ctx.GetTemp("name","").(string)//粉丝数fansNum := ctx.GetTemp("fansNum","").(string)//地区city := ctx.GetTemp("city","").(string)//微博数weiboNum := ctx.GetDom().Find(".tc").Text()j := strings.LastIndex(weiboNum,"[")z := strings.LastIndex(weiboNum,"]")weiboNum = weiboNum[j+1:z]//关注数attentionNum := ctx.GetDom().Find(".tip2 a").Eq(0).Text()j = strings.LastIndex(attentionNum,"[")z = strings.LastIndex(attentionNum,"]")attentionNum = attentionNum[j+1:z]a :=ctx.GetDom().Find(".ut a").Eq(1)if a.Text() == "加关注"{if url, ok := ctx.GetDom().Find(".ut a").Eq(3).Attr("href"); ok {ctx.AddQueue(&request.Request{Url:  "https://weibo.cn" + url,Rule: "资料页",EnableCookie: true,Temp: map[string]interface{}{"name":  name,"fansNum": fansNum,"city":  city,"weiboNum":  weiboNum,"attentionNum":  attentionNum,},})}} else{if url, ok := ctx.GetDom().Find(".ut a").Eq(2).Attr("href"); ok {ctx.AddQueue(&request.Request{Url:  "https://weibo.cn" + url,Rule: "资料页",EnableCookie: true,Temp: map[string]interface{}{"name":  name,"fansNum": fansNum,"city":  city,"weiboNum":  weiboNum,"attentionNum":  attentionNum,},})}}},},"资料页": {ItemFields: []string{"昵称","粉丝数","地区","微博数","关注数","标签","详细信息",},ParseFunc: func(ctx *Context) {//昵称name := ctx.GetTemp("name","").(string)//粉丝数fansNum := ctx.GetTemp("fansNum","").(string)//地区city := ctx.GetTemp("city","").(string)//微博数weiboNum := ctx.GetTemp("weiboNum","").(string)//关注数attentionNum := ctx.GetTemp("attentionNum","").(string)str := ctx.GetDom().Find("div").Eq(5).Text()i := strings.LastIndex(str,"标签")z := strings.LastIndex(str,"更多")var str2,str3 stringif i == -1{str2 = ""str3 = str}else{//标签str2 = str[i+7:z]//详细信息str3 = str[0:i]}ctx.Output(map[int]interface{}{0: name,1: fansNum,2: city,3: weiboNum,4: attentionNum,5: str2,6: str3,})},},},},
}

//获取unix时间
func  getMillisecond() int64{MS := time.Now().UnixNano()/1000return MS
}//用户名base64加密
func encryptUname(uname string) string {  // 获取username base64加密后的结果//println(base64.RawURLEncoding.EncodeToString([]byte(uname)))return base64.URLEncoding.EncodeToString([]byte(uname))
}//密码加密
//把字符串转换bigint
func string2big(s string) *big.Int {ret := new(big.Int)ret.SetString(s, 16)  // 将字符串转换成16进制return ret
}func encryptPassword(pubkey string,servertime string,nonce string, password string) string{pub := rsa.PublicKey{N: string2big(pubkey),E: 65537,                       // 10001是十六进制数，65537是它的十进制表示}// servertime、nonce之间加\t，然后在\n ,和password拼接encryString := servertime + "\t" + nonce + "\n" + password// 拼接字符串加密encryResult, _ := rsa.EncryptPKCS1v15(rand.Reader, &pub, []byte(encryString))return hex.EncodeToString(encryResult)
}