import com.sun.tools.doclets.formats.html.SourceToHTMLConverter;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;/*** Created with Chenquan.* Description: 淘宝抓包* Date: 2018-12-13* Time: 15:12*/
public class TaobaoCatch {public static void main(String[] args) {int i = 0;/*        String url = "https://acs.m.taobao.com/h5/mtop.taobao.wsearch.h5search/1.0/?jsv=2.3.16&appKey=12574478&t=1545023581359&sign=e3476c9041a75de0a9190da470204d93&api=mtop.taobao.wsearch.h5search&v=1.0&H5Request=true&ecode=1&type=jsonp&dataType=jsonp&callback=mtopjsonp1&data=%7B%22q%22%3A%22%E4%BB%99%E6%B6%B5%E5%86%85%E8%A1%A3%22%2C%22search%22%3A%22%E6%8F%90%E4%BA%A4%22%2C%22tab%22%3A%22all%22%2C%22sst%22%3A%221%22%2C%22n%22%3A20%2C%22buying%22%3A%22buyitnow%22%2C%22m%22%3A%22api4h5%22%2C%22token4h5%22%3A%22%22%2C%22abtest%22%3A%221%22%2C%22wlsort%22%3A%221%22%2C%22page%22%3A1%7D";Connection con = Jsoup.connect(url);con.header("Cookie", "cna=TA+aFFGXQFUCAXQaRYGZVU8Q; t=efa81a9785cd86f885e13998b6d5f9cb; thw=cn; uc3=vt3=F8dByRzMU9X8Hvccr00%3D&id2=W8zLpWipxVFu&nk2=0PLo6GHZOM8%3D&lg2=V32FPkk%2Fw0dUvg%3D%3D; tracknick=%5Cu9648%5Cu94E81992; lgc=%5Cu9648%5Cu94E81992; _cc_=Vq8l%2BKCLiw%3D%3D; tg=0; enc=4rB%2FfKFx8DJKgPpoHlZjr824CEYw%2BlPaKBDWbFO4fnh6svGA97NoZNGERui4fOo2tXSnSVN1ygkfn5R5ekztTQ%3D%3D; hng=CN%7Czh-CN%7CCNY%7C156; mt=ci=0_1; _m_h5_tk=e501ac7690832934d663aef19ee36be5_1545033419107; _m_h5_tk_enc=5147579a652b4fb508dc886d59c37045; isg=BFVVgDOkpYNz64H7Z31pC9thZFHP-goqhI4h7tf6EUwbLnUgn6IZNGPv_DSYLiEc");
//        con.header("referer", "https://item.taobao.com/item.htm ");Connection.Response resp = null;try {resp = con.method(Connection.Method.GET).ignoreContentType(true).execute();} catch (IOException e) {e.printStackTrace();}String body = resp.body();
//        System.out.println(body);body = body.substring(12, body.length() - 1);JSONObject jb = JSONObject.fromObject(body);JSONArray jsonArray = jb.getJSONObject("data").getJSONArray("listItem");//        while(i<100){i++;for (int j = 0; j < jsonArray.size(); j++) {JSONObject jsonObject = jsonArray.getJSONObject(j);String item_id = jsonObject.getString("item_id");System.out.println("item_id: "+item_id);getAll(item_id);}
//        }*///传链接上的产品idgetAll("577996531297");}public static void getAll(String item_id ) {try {Thread.sleep(2000);//一个休息5s，太快会被禁} catch (InterruptedException e) {e.printStackTrace();}System.out.println("开始时间：" + new Date());Date dateStart = new Date();Document doc = null;String id = "";try {
//            int i = 0;
//            while(i < 10000){
//            i++;String url = "https://item.taobao.com/item.htm?id="+item_id;id = getParamByUrl(url, "id");doc = Jsoup.connect(url).ignoreContentType(true).get();/* String url = "https://h5api.m.taobao.com/h5/mtop.taobao.detail.getdetail/6.0/?data=";//手机的html 5 页面 ，为了获取库存、价格String enc = "{\"itemNumId\":\"582061497975\"}";String gbk = URLEncoder.encode(enc, "utf-8");String sds = url + gbk;System.out.println("库存、价格"+sds);doc = Jsoup.connect(sds).ignoreContentType(true).get();*///设置请求头
//                    Connection con = Jsoup.connect(url);
//                    con.header("Cookie", " enc=1LWJWtPGgf6MF1NVsn2rbeRb3%2FU1%2Fk5ZiiedHbVedmxmfvUUWDPmFeyKeLYl7NVchBB19JCIVnX0eFv4otK9HA%3D%3D;" +
//                            "x5sec=7b2264657461696c736b69703b32223a226235653133353933646637396131353230343663346139633633653038326465434c6a4e7a654146454e447739724732716644534b426f4c4f4455774d7a51304e7a4d794f7a453d227d;" );
//                    con.header("referer", "https://item.taobao.com/item.htm ");
//                    Connection.Response resp=con.method(Connection.Method.GET).execute();
//                    Map<String,String> cookies = resp.cookies();
//                    Connection.Request request = con.request();
//                    String body = resp.body();
} catch (IOException e) {e.printStackTrace();}if (doc.baseUri().contains("tmall")) {System.out.println("商品名称："+ doc.select("h1[data-spm=\"1000983\"]").text());}else {System.out.println("商品名称：" + doc.select("h3[class=\"tb-main-title\"]").text());}Elements imgSrcElement = doc.select("#J_UlThumb > li");for (Element element : imgSrcElement) {String imgSrc = "";if (element.baseUri().contains("tmall")){imgSrc = element.getElementsByTag("img").attr("src");}else{imgSrc = element.getElementsByTag("img").attr("data-src");}imgSrc = imgSrc.replaceFirst("//img.alicdn.com/imgextra/", "");imgSrc = imgSrc.substring(0, imgSrc.length() - 10);
//            imgSrc = imgSrc.replaceAll("_60x60q90.jpg",""); //处理掉不必要的数据System.out.println("主图url：" + imgSrc);}// 规格参数Elements selectRules = doc.select(".J_TSaleProp");List<List<String>> liHashMap = new ArrayList<>();for (Element ulElement : selectRules) {String ul = ulElement.getElementsByTag("ul").attr("data-property");System.out.println("ul：" + ul);List<String> liString = new ArrayList<>();for (Element liElement : ulElement.getElementsByTag("li")) {String liDataValue = liElement.getElementsByTag("li").attr("data-value");System.out.println("liDataValue: " + liDataValue);liString.add(liDataValue);String aStyle = liElement.getElementsByTag("a").attr("style");if (StringUtils.isNotBlank(aStyle)) {aStyle = aStyle.replaceAll("background:url\\(", "");aStyle = aStyle.substring(0, aStyle.length() - 29);
//                aStyle = aStyle.replaceAll("_40x40q90.jpg\\) center no-repeat;", "");System.out.println("aStyle: " + aStyle);}String spanText = liElement.getElementsByTag("span").text();if (StringUtils.isNotBlank(spanText)) {System.out.println("spanText: " + spanText);}}liHashMap.add(liString);}List<String> combination = test.combination(liHashMap);//获取价格、库存Elements eles = doc.getElementsByTag("script");for (Element ele : eles) {String s = ele.toString();if (!ele.baseUri().contains("tmall")) {//淘宝String rgex = "";String subUtilSimple = "";if (s.contains("skuMap")) {//获取sku的idrgex = "skuMap(.*?)propertyMemoMap";String skuId = s.replaceAll("\\s*", "");//                System.out.println(s);subUtilSimple = getSubUtilSimple(skuId, rgex);subUtilSimple = subUtilSimple.substring(1, subUtilSimple.length() - 1);//
                    JSONObject jb = JSONObject.fromObject(subUtilSimple);JSONObject finalJb = jb;List<String> skuList = new ArrayList<>();combination.forEach(p->{JSONObject jsonObject = finalJb.getJSONObject(";" + p + ";");if (!jsonObject.isNullObject()) {String o = jsonObject.getString("skuId");System.out.println("sku的id: " + o);skuList.add(o);}});String url = "https://h5api.m.taobao.com/h5/mtop.taobao.detail.getdetail/6.0/?data=";//手机的html 5 页面 ，为了获取库存、价格String enc = "{\"itemNumId\":\"" + id + "\"}";String substore = "";String store = "";try {String gbk = URLEncoder.encode(enc, "utf-8");String sds = url + gbk;System.out.println("库存、价格" + sds);doc = Jsoup.connect(sds).ignoreContentType(true).get();store = doc.toString();rgex = "sku2info(.*?)skuItem";substore = getSubUtilSimple(store, rgex);substore = substore.substring(3, substore.length() - 3);String sub = substore.replaceAll("\\\\", "").replaceAll("\\s*", "");JSONObject sb = JSONObject.fromObject(sub);skuList.stream().forEach(p->{if (sb.has(p)) {//判断是否有值，没值不取，不然会报错String string = sb.getString(p);System.out.println("淘宝的价格库存==============" + string);}});} catch (Exception e) {System.out.println("报错的地方store：" + store);
//                        System.out.println("报错的地方substore：" + substore);
                        e.printStackTrace();System.out.println("=====================================程序报错，提前结束===================================================" );return;}}if (s.contains("descUrl") && s.contains("counterApi")) {//                System.out.println(s);//详情链接rgex = "protocol(.*?)desc\\.alicdn\\.com";subUtilSimple = getSubUtilSimple(s, rgex);subUtilSimple = subUtilSimple.substring(14, subUtilSimple.length() - 7);System.out.println("详情链接: " + subUtilSimple);try {doc = Jsoup.connect("http:" + subUtilSimple).get();} catch (IOException e) {e.printStackTrace();}Elements imgDetail = doc.getElementsByTag("img");for (Element element : imgDetail) {String imgSrc = element.getElementsByTag("img").attr("src");//                    imgSrc = imgSrc.replaceFirst("//img.alicdn.com/imgextra/","");//                    imgSrc = imgSrc.replaceAll("_60x60q90.jpg",""); //处理掉不必要的数据if (StringUtils.isNotBlank(imgSrc)) {System.out.println("详情图url：" + imgSrc);}}}}else {//天猫的if (s.contains("TShop.Setup")) {//                String rgex = "<bdocid>(.*?)</bdocid>";String rgex = "skuMap(.*?)salesProp";String subUtilSimple = getSubUtilSimple(s, rgex);subUtilSimple = subUtilSimple.substring(2, subUtilSimple.length() - 2);JSONObject jb = JSONObject.fromObject(subUtilSimple);List<String> skuList = new ArrayList<>();combination.forEach(p->{JSONObject jsonObject = jb.getJSONObject(";" + p + ";");if (!jsonObject.isNullObject()) {String skuId = jsonObject.getString("skuId");System.out.println(skuId);skuList.add(skuId);}});//库存、价格String url = "https://h5api.m.taobao.com/h5/mtop.taobao.detail.getdetail/6.0/?data=";//手机的html 5 页面 ，为了获取库存、价格String enc = "{\"itemNumId\":\"" + id + "\"}";String substore = "";String store = "";try {String gbk = URLEncoder.encode(enc, "utf-8");String sds = url + gbk;System.out.println("库存、价格" + sds);doc = Jsoup.connect(sds).ignoreContentType(true).get();store = doc.toString();rgex = "sku2info(.*?)skuItem";substore = getSubUtilSimple(store, rgex);substore = substore.substring(3, substore.length() - 3);String sub = substore.replaceAll("\\\\", "").replaceAll("\\s*", "");JSONObject sb = JSONObject.fromObject(sub);skuList.stream().forEach(p->{if (sb.has(p)) {//判断是否有值，没值不取，不然会报错String string = sb.getString(p);System.out.println("天猫的价格库存==============" + string);}});} catch (Exception e) {System.out.println("报错的地方store：" + store);
//                        System.out.println("报错的地方substore：" + substore);
                        e.printStackTrace();System.out.println("=====================================程序报错，提前结束===================================================" );return;}//详情链接rgex = "httpsDescUrl(.*?)fetchDcUrl";subUtilSimple = getSubUtilSimple(s, rgex);subUtilSimple = subUtilSimple.substring(3, subUtilSimple.length() - 3);System.out.println(subUtilSimple);try {doc = Jsoup.connect("http:"+subUtilSimple).get();} catch (IOException e) {e.printStackTrace();}Elements imgDetail = doc.getElementsByTag("img");for (Element element :imgDetail) {String imgSrc = element.getElementsByTag("img").attr("src");
//                    imgSrc = imgSrc.replaceFirst("//img.alicdn.com/imgextra/","");
//                    imgSrc = imgSrc.replaceAll("_60x60q90.jpg",""); //处理掉不必要的数据System.out.println("详情图url："+imgSrc);}break;}}}System.out.println("结束时间：" + new Date());Date dateEnd = new Date();long number = dateEnd.getTime()-dateStart.getTime();//然后在将毫秒转换为date类型就可以了System.out.println("时间差为： "+number/1000);}/*** 返回单个字符串，若匹配到多个的话就返回第一个，方法与getSubUtil一样** @param soap* @param rgex* @return*/public static String getSubUtilSimple(String soap, String rgex) {Pattern pattern = Pattern.compile(rgex);// 匹配的模式Matcher m = pattern.matcher(soap);while (m.find()) {return m.group(1);}return "";}/*** 获取指定url中的某个参数** @param url* @param name* @return*/public static String getParamByUrl(String url, String name) {url += "&";String pattern = "(\\?|&){1}#{0,1}" + name + "=[a-zA-Z0-9]*(&{1})";Pattern r = Pattern.compile(pattern);Matcher m = r.matcher(url);if (m.find()) {
//            System.out.println(m.group(0));return m.group(0).split("=")[1].replace("&", "");} else {return null;}}}

import com.google.gson.JsonObject;
import net.sf.json.JSONObject;import java.util.ArrayList;
import java.util.List;/*** Created with Chenquan.* Description:* Date: 2018-12-16* Time: 10:27*/
public class test {public static void main(String[] args) {List<String> li = new ArrayList<>();li.add("aa");li.add("bb");li.add("cc");List<String> bi = new ArrayList<>();bi.add("ee");bi.add("rr");bi.add("tt");List<String> ci = new ArrayList<>();ci.add("yy");ci.add("uu");ci.add("ii");List<List<String>> list = new ArrayList<>();list.add(li);list.add(bi);list.add(ci);List<String> vv = new ArrayList<>();List<String> combination = combination(list);System.out.println(combination);}/*** 若干个集合元素的组合** @param groups 多个集合* @return 组合结果*/public static List<String> combination(List<List<String>> groups) {if (invalid(groups) || invalid(groups.get(0))) {return null;}List<String> combine = groups.get(0);for (int i = 1; i < groups.size(); i++) {combine = cartesianProduct(combine, groups.get(i));if (combine == null) {return null;}}return combine;}/*** 两个集合元素的组合** @param c1 集合1* @param c2 集合2* @return 组合结果*/public static List<String> cartesianProduct(List<String> c1, List<String> c2) {if (invalid(c1) || invalid(c2)) {return null;}List<String> combine = new ArrayList<>();for (String s : c1) {for (String t : c2) {combine.add(String.format("%s;%s", s, t));//combine.add(String.format("%s%s", t, s));}}return combine;}/*** 验证集合是否无效** @param c 集合* @return true 无效*/private static boolean invalid(List<?> c) {return c == null || c.isEmpty();}}