package 虎牙斗鱼直播平台;

import java.io.IOException;

import java.sql.DriverManager;

import java.sql.PreparedStatement;

import java.sql.SQLException;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import org.jsoup.Connection;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.select.Elements;

import com.alibaba.fastjson.JSON;

import com.alibaba.fastjson.JSONArray;

import com.alibaba.fastjson.JSONObject;

import com.gargoylesoftware.htmlunit.BrowserVersion;

import com.gargoylesoftware.htmlunit.WebClient;

import com.gargoylesoftware.htmlunit.html.HtmlAnchor;

import com.gargoylesoftware.htmlunit.html.HtmlPage;

public class 抓取斗鱼 {

public static void main(String[] args) throws IOException, SQLException, ClassNotFoundException {

int id=1;

java.sql.Connection con;

Class.forName("com.mysql.jdbc.Driver");

System.out.println("数据库驱动加载成功");

con = DriverManager.getConnection("jdbc:mysql:"

+ "//127.0.0.1:3306/db?characterEncoding=utf-8", "root", "123456");

System.out.println("数据库连接成功");

final WebClient webClient = new WebClient(BrowserVersion.CHROME);//

webClient.getOptions().setJavaScriptEnabled(true);

webClient.getOptions().setThrowExceptionOnScriptError(false);

webClient.getOptions().setCssEnabled(false);

HtmlPage page=webClient.getPage("https://www.douyu.com/directory/all");

Listanchors=page.getAnchors();

Listjsanchors=new ArrayList();

for(HtmlAnchor anchor:anchors)

{ String href=anchor.getAttribute("href");

/*

* 抓取js的链接 然后获取有用的信息

*/

if("#".equals(href))

{

{jsanchors.add(anchor);}

}

}

HtmlAnchor t1=jsanchors.get(jsanchors.size()-3);System.out.println(t1.asText());//加载js才能获取页面数

int number=Integer.parseInt(t1.asText());//页面数

for(int i=1;i

String url="https://www.douyu.com/gapi/rkc/directory/0_0/"+i;

Connection conn = Jsoup.connect(url).ignoreContentType(true).timeout(2000);

// .header("Accept", "text/plain, */*; q=0.01")

// .header("Accept-Encoding", "gzip, deflate, br")

// .header("Accept-Language", "zh-CN,zh;q=0.9")

// .header("Connection", "keep-alive")

// .header("content type", " text/*, application/xml, or application/xhtml+xml")

// .header("Mimetype", "application/json")

// .header("charset", "utf-8")

// .header("X-Requested-With", "XMLHttpRequest")

// .header("Host", "www.douyu.com")

// .header("Cookie", "dy_did=1c44f89110e75337d1bb4ab500011501; acf_did=1c44f89110e75337d1bb4ab500011501;"

// + " smidV2=2018042210064794358e4a60e148cb2044dd7ccc979fb90028d0af372db9810; "

// + "Hm_lvt_e99aee90ec1b2106afe7ec3b199020a7=1524114219,1524218776,1524383249,1524660154;"

// + " Hm_lpvt_e99aee90ec1b2106afe7ec3b199020a7=1524660429")

// .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36");

Document doc=conn.get();

// System.out.print(doc);

// int page=0;//页数

try{

JSONObject jsonObj = JSON.parseObject(doc.text());

JSONObject jsonOb=JSON.parseObject(jsonObj.getString("data"));//获取data的json内容

JSONArray jarr=jsonOb.getJSONArray("rl");//

for(Object jar:jarr)

{

JSONObject js=JSON.parseObject( jar.toString());//获取更具体的json值

//System.out.println(js);

/*

* 写入数据库

*/

int renshu2=Integer.parseInt(js.get("ol").toString());

String type=js.get("c2name").toString();

String sql2="insert into satuday(name,href,number,type)value(?,?,?,?)";//插入mysql

PreparedStatement pstmt = con.prepareStatement(sql2);

pstmt.setString(1, (String) js.get("nn"));

pstmt.setString(2, "https://www.douyu.com"+js.get("url"));

pstmt.setInt(3, renshu2);

pstmt.setString(4,type );

pstmt.execute();

System.out.println(js.get("nn")+""+id+++""+type+renshu2);

}

}

catch(Exception e) {System.out.println(e.getMessage());}

}

con.close();

}

}