import requests

import json

import pandas as pd

import time

# getRtVisitor.json

session = requests.Session() # 创建一个session对象

headers = {

'accept':'*/*',

'accept-encoding':'gzip, deflate, br',

'accept-language':'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',

'cookie':'t=4a09a3c800e9591a2c198b124d80e02a; cna=G2veFsG5YX0CAQ5pIwfTWHFB; lgc=%5Cu5927%5Cu5468%5Cu4F73; tracknick=%5Cu5927%5Cu5468%5Cu4F73; tg=0; _euacm_ac_l_uid_=1639181234; 1639181234_euacm_ac_c_uid_=1639181234; 1639181234_euacm_ac_rs_uid_=1639181234; _portal_version_=new; cc_gray=1; thw=cn; mt=ci=84_1; _euacm_ac_rs_sid_=155550734; enc=Rk6EFG1Zi%2F5sBqHEJADmGhcHg%2F8HVbpOVQDb72MK8zmamaAE7C23fINFdlW5BT%2FNfkRKZLPkB8gKw%2Bj0cK0hig%3D%3D; hng=CN%7Czh-CN%7CCNY%7C156; cookie2=1a5b17269a2a570500957e7dcce65c45; _tb_token_=35e1e66785eb7; _samesite_flag_=true; sgcookie=Ep88649ag1i3VPa4P6akw; unb=1639181234; uc3=vt3=F8dBxd9nWWLAbiRWGZE%3D&lg2=UtASsssmOIJ0bQ%3D%3D&nk2=1z8nwQRn&id2=Uoe3fo1fFq44Zg%3D%3D; csg=aa9633c3; cookie17=Uoe3fo1fFq44Zg%3D%3D; dnk=%5Cu5927%5Cu5468%5Cu4F73; skt=69bf89596c5fe0d1; existShop=MTU4NDU4MTc0MQ%3D%3D; uc4=nk4=0%401fDckZcjfHBEZVI1NQCO3RY%3D&id4=0%40UO%2BxIxkhtoiLbBQVUeEnpSn1KHbH; _cc_=U%2BGCWk%2F7og%3D%3D; _l_g_=Ug%3D%3D; sg=%E4%BD%B34e; _nk_=%5Cu5927%5Cu5468%5Cu4F73; cookie1=BYXJ7V2Aq8c%2FnceEFKLACXxZ7qw2VwJYwgQbxY%2Feb6A%3D; tfstk=cKiNBsfOWhKwV3THPur4G7Z13THOZAHinMy7S4xicwaWPJ4GiQfYKvNUL-NJxPf..; v=0; _m_h5_tk=8b49ef5c39700daef25a92f8c520cc65_1584590383214; _m_h5_tk_enc=4d9374ef7e4743b408cb6247793448be; uc1=cookie14=UoTUPvXUU0lRTg%3D%3D&lng=zh_CN&cookie16=Vq8l%2BKCLySLZMFWHxqs8fwqnEw%3D%3D&existShop=true&cookie21=UtASsssmfavZrexPkAwn7A%3D%3D&tag=8&cookie15=W5iHLLyFOGW7aA%3D%3D&pas=0; XSRF-TOKEN=673a0ad9-3e96-4fb2-bbd9-af3ffa50a9bc; JSESSIONID=B0A57AC70F717D1231606D512F674ECB; l=dBjCTtrcQuukdAdWBOfgqK_ahxbOrIRbzsPPlS9CCICP9O1wrJp1WZ4V8-8eCnGN36J6R3WhGKB3BqLTFPaOhtikBBrsDOsCydTBR; isg=BAoK5gRhYDd8MOy0NcE2jJ5BW_Cs-45VYmG9hZRDct30R6gBfIkRZ2NxV7ubtwbt',

'referer':'https://sycm.taobao.com/ipoll/visitor.htm?spm=a21ag.7622617.LeftMenu.d181.758a1be9MfheaI',

'sec-fetch-mode':'cors',

'sec-fetch-site':'same-origin',

'user-agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36'

}

def loadPage(page):

strPage=str(page)

strToken='6f5921e86'

strLastPara='1584581855701'

url = 'https://sycm.taobao.com/ipoll/live/visitor/getRtVisitor.json?device=2&limit=20&page='+strPage+'&token='+strToken+'&type=Y&_='+strLastPara

# https://sycm.taobao.com/ipoll/live/visitor/getRtVisitor.json?device=2&limit=20&page=2&token=97890b4c6&type=Y&_=1584409617907

req = session.get(url, headers=headers) #发起get请求

text = req.text

with open(str(page)+".txt", "w") as f:

f.write(text)

return json.loads(text)

def GetValueFromDic(item, key, defaultValue=None):

if key in item:

return item[key]

if defaultValue is None:

return ''

return defaultValue

def appendList(json_res):

list_res = json_res['data']['data']['list']

for item in list_res:

arr=[]

if item['pageTypeId'] == 5:# 商品分类页

arr.append(item['visitTime'])

arr.append('其它来源')

arr.append('')# 搜索关键字

arr.append('商品分类页')

arr.append('')# 访客位置

arr.append(item['oid'])

else:

arr.append(GetValueFromDic(item, 'visitTime'))

arr.append(GetValueFromDic(item, 'srcGrpName', '其它来源'))

arr.append(GetValueFromDic(item, 'preSeKeyword'))

arr.append(GetValueFromDic(item, 'title'))

arr.append(GetValueFromDic(item, 'cityName'))

arr.append(GetValueFromDic(item, 'oid'))

list_excel.append(arr)

# list集合

list_excel=[]

# 数据源

json_res = loadPage(1)

# 解析数据

totalPage = json_res['data']['data']['totalPage']

curPage = json_res['data']['data']['page']

print('总页数:', totalPage, '当前页:', curPage)

# 追加数据

appendList(json_res)

# 循环下载

for i in range(totalPage-1):

page = i + 2

print('当前页:', page)

json_res = loadPage(page)

appendList(json_res)

# 转换为DataFrame

#df = pd.DataFrame(list_excel, columns=['visitTime', 'srcGrpName', 'preSeKeyword', 'title', 'cityName', 'oid'])

df = pd.DataFrame(list_excel, columns=['访问时间', '入店来源', '搜索关键字', '被访页面', '访客位置', '访客编号'])

# 保存到本地excel

curTime = time.strftime('%Y-%m-%d %H%M%S',time.localtime(time.time()))

df.to_excel('访客-'+curTime+'.xlsx', index=False)

mark

pageTypeId：访客访问页面的类型

具体看这里：item['pageTypeId'] == 5:# 商品分类页

pageTypeId==5，表示用户是从商品分类页来的，或者点了商品分类页页