#!/usr/bin/env python# encoding=utf-8from __future__ import print_function

import osimport requestsimport reimport timeimport xml.dom.minidomimport jsonimport sysimport mathimport subprocessimport sslimport threadingimport urllib, urllib2

DEBUG = False

MAX_GROUP_NUM = 2  # 每组人数INTERFACE_CALLING_INTERVAL = 5  # 接口调用时间间隔, 间隔太短容易出现"操作太频繁", 会被限制操作半小时左右MAX_PROGRESS_LEN = 50

QRImagePath = os.path.join(os.getcwd(), 'qrcode.jpg')

tip = 0uuid = ''

base_uri = ''redirect_uri = ''push_uri = ''

skey = ''wxsid = ''wxuin = ''pass_ticket = ''deviceId = 'e000000000000000'

BaseRequest = {}

ContactList = []My = []SyncKey = []

try:    xrange    range = xrangeexcept:    # python 3    pass

def responseState(func, BaseResponse):    ErrMsg = BaseResponse['ErrMsg']    Ret = BaseResponse['Ret']    if DEBUG or Ret != 0:        print('func: %s, Ret: %d, ErrMsg: %s' % (func, Ret, ErrMsg))

    if Ret != 0:        return False

    return True

def getUUID():    global uuid

    url = 'https://login.weixin.qq.com/jslogin'    params = {        'appid': 'wx782c26e4c19acffb','fun': 'new','lang': 'zh_CN','_': int(time.time()),}

    r = myRequests.get(url=url, params=params)    r.encoding = 'utf-8'    data = r.text

# print(data)

    # window.QRLogin.code = 200; window.QRLogin.uuid = "oZwt_bFfRg==";    regx = r'window.QRLogin.code = (\d+); window.QRLogin.uuid = "(\S+?)"'    pm = re.search(regx, data)

    code = pm.group(1)    uuid = pm.group(2)

    if code == '200':        return True

    return False

def showQRImage():    global tip

    url = 'https://login.weixin.qq.com/qrcode/' + uuid    params = {        't': 'webwx','_': int(time.time()),}

    r = myRequests.get(url=url, params=params)    tip = 1

    f = open(QRImagePath, 'wb+')    f.write(r.content)    f.close()    time.sleep(1)

    if sys.platform.find('darwin') >= 0:        subprocess.call(['open', QRImagePath])    else:        os.startfile(QRImagePath)    print('请使用微信扫描二维码以登录')

def waitForLogin():    global tip, base_uri, redirect_uri, push_uri

    url = 'https://login.weixin.qq.com/cgi-bin/mmwebwx-bin/login?tip=%s&uuid=%s&_=%s' % (        tip, uuid, int(time.time()))

    r = myRequests.get(url=url)    r.encoding = 'utf-8'    data = r.text

# print(data)

    # window.code=500;    regx = r'window.code=(\d+);'    pm = re.search(regx, data)

    code = pm.group(1)

    if code == '201':  # 已扫描        print('成功扫描,请在手机上点击确认以登录')        tip = 0    elif code == '200':  # 已登录        print('正在登录...')        regx = r'window.redirect_uri="(\S+?)";'        pm = re.search(regx, data)        redirect_uri = pm.group(1) + '&fun=new'        base_uri = redirect_uri[:redirect_uri.rfind('/')]

        # push_uri与base_uri对应关系(排名分先后)(就是这么奇葩..)        services = [            ('wx2.qq.com', 'webpush2.weixin.qq.com'),('qq.com', 'webpush.weixin.qq.com'),('web1.wechat.com', 'webpush1.wechat.com'),('web2.wechat.com', 'webpush2.wechat.com'),('wechat.com', 'webpush.wechat.com'),('web1.wechatapp.com', 'webpush1.wechatapp.com'),]        push_uri = base_urifor (searchUrl, pushUrl) in services:            if base_uri.find(searchUrl) >= 0:                push_uri = 'https://%s/cgi-bin/mmwebwx-bin' % pushUrlbreak

        # closeQRImage        if sys.platform.find('darwin') >= 0:  # for OSX with Preview            os.system("osascript -e 'quit app \"Preview\"'")    elif code == '408':  # 超时        pass    # elif code == '400' or code == '500':

    return code

def login():    global skey, wxsid, wxuin, pass_ticket, BaseRequest

    r = myRequests.get(url=redirect_uri)    r.encoding = 'utf-8'    data = r.text

# print(data)

    doc = xml.dom.minidom.parseString(data)    root = doc.documentElement

for node in root.childNodes:        if node.nodeName == 'skey':            skey = node.childNodes[0].dataelif node.nodeName == 'wxsid':            wxsid = node.childNodes[0].dataelif node.nodeName == 'wxuin':            wxuin = node.childNodes[0].dataelif node.nodeName == 'pass_ticket':            pass_ticket = node.childNodes[0].data

# print('skey: %s, wxsid: %s, wxuin: %s, pass_ticket: %s' % (skey, wxsid,    # wxuin, pass_ticket))

    if not all((skey, wxsid, wxuin, pass_ticket)):        return False

    BaseRequest = {        'Uin': int(wxuin),'Sid': wxsid,'Skey': skey,'DeviceID': deviceId,}

    return True

def webwxinit():    url = (base_uri +           '/webwxinit?pass_ticket=%s&skey=%s&r=%s' % (               pass_ticket, skey, int(time.time())))    params = {'BaseRequest': BaseRequest}    headers = {'content-type': 'application/json; charset=UTF-8'}

    r = myRequests.post(url=url, data=json.dumps(params), headers=headers)    r.encoding = 'utf-8'    data = r.json()

    if DEBUG:        f = open(os.path.join(os.getcwd(), 'webwxinit.json'), 'wb')        f.write(r.content)        f.close()

    # print(data)

    global ContactList, My, SyncKey    dic = data    ContactList = dic['ContactList']    My = dic['User']    SyncKey = dic['SyncKey']

    state = responseState('webwxinit', dic['BaseResponse'])    return state

def webwxgetcontact():    url = (base_uri +           '/webwxgetcontact?pass_ticket=%s&skey=%s&r=%s' % (               pass_ticket, skey, int(time.time())))    headers = {'content-type': 'application/json; charset=UTF-8'}

    r = myRequests.post(url=url, headers=headers)    r.encoding = 'utf-8'    data = r.json()

    if DEBUG:        f = open(os.path.join(os.getcwd(), 'webwxgetcontact.json'), 'wb')        f.write(r.content)        f.close()

    dic = data    MemberList = dic['MemberList']

    # 倒序遍历,不然删除的时候出问题..    SpecialUsers = ["newsapp", "fmessage", "filehelper", "weibo", "qqmail", "tmessage", "qmessage", "qqsync","floatbottle", "lbsapp", "shakeapp", "medianote", "qqfriend", "readerapp", "blogapp", "facebookapp","masssendapp","meishiapp", "feedsapp", "voip", "blogappweixin", "weixin", "brandsessionholder", "weixinreminder","wxid_novlwrv3lqwv11", "gh_22b87fa7cb3c", "officialaccounts", "notification_messages", "wxitil","userexperience_alarm"]    for i in range(len(MemberList) - 1, -1, -1):        Member = MemberList[i]        if Member['VerifyFlag'] & 8 != 0:  # 公众号/服务号            MemberList.remove(Member)        elif Member['UserName'] in SpecialUsers:  # 特殊账号            MemberList.remove(Member)        elif Member['UserName'].find('@@') != -1:  # 群聊            MemberList.remove(Member)        elif Member['UserName'] == My['UserName']:  # 自己            MemberList.remove(Member)

    return MemberList

def syncKey():    SyncKeyItems = ['%s_%s' % (item['Key'], item['Val'])                    for item in SyncKey['List']]    SyncKeyStr = '|'.join(SyncKeyItems)    return SyncKeyStr

def syncCheck():    url = push_uri + '/synccheck?'    params = {        'skey': BaseRequest['Skey'],'sid': BaseRequest['Sid'],'uin': BaseRequest['Uin'],'deviceId': BaseRequest['DeviceID'],'synckey': syncKey(),'r': int(time.time()),}

    r = myRequests.get(url=url, params=params)    r.encoding = 'utf-8'    data = r.text

# print(data)

    # window.synccheck={retcode:"0",selector:"2"}    regx = r'window.synccheck={retcode:"(\d+)",selector:"(\d+)"}'    pm = re.search(regx, data)

    retcode = pm.group(1)    selector = pm.group(2)

    return selector

def webwxsync():    global SyncKey

    url = base_uri + '/webwxsync?lang=zh_CN&skey=%s&sid=%s&pass_ticket=%s' % (        BaseRequest['Skey'], BaseRequest['Sid'], urllib.quote_plus(pass_ticket))    params = {        'BaseRequest': BaseRequest,'SyncKey': SyncKey,'rr': ~int(time.time()),}    headers = {'content-type': 'application/json; charset=UTF-8'}

    r = myRequests.post(url=url, data=json.dumps(params))    r.encoding = 'utf-8'    data = r.json()

    # print(data)

    dic = data    SyncKey = dic['SyncKey']

    state = responseState('webwxsync', dic['BaseResponse'])    return state

def heartBeatLoop():    while True:        selector = syncCheck()        if selector != '0':            webwxsync()        time.sleep(1)

def main():    global myRequests

if hasattr(ssl, '_create_unverified_context'):        ssl._create_default_https_context = ssl._create_unverified_context

    headers = {        'User-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36'}    myRequests = requests.Session()    myRequests.headers.update(headers)

    if not getUUID():        print('获取uuid失败')        return

    print('正在获取二维码图片...')    showQRImage()

    while waitForLogin() != '200':        pass

    os.remove(QRImagePath)

    if not login():        print('登录失败')        return

    if not webwxinit():        print('初始化失败')        return

    MemberList = webwxgetcontact()

    threading.Thread(target=heartBeatLoop)

    MemberCount = len(MemberList)    print('通讯录共%s位好友' % MemberCount)

    d = {}    imageIndex = 0    for Member in MemberList:        imageIndex = imageIndex + 1        # name = 'C:\\Users\\Public\\Pictures\\' + str(imageIndex) + '.jpg'        # imageUrl = 'http://wx2.qq.com' + Member['HeadImgUrl']        # r = myRequests.get(url=imageUrl, headers=headers)        # imageContent = (r.content)        # fileImage = open(name, 'wb')        # fileImage.write(imageContent)        # fileImage.close()        # print('正在下载第：' + str(imageIndex) + '位好友头像')        d[Member['UserName']] = (Member['NickName'], Member['RemarkName'])        city = Member['City']        city = 'nocity' if city == '' else  city        name = Member['NickName']        name = 'noname' if name == '' else  name        sign = Member['Signature']        sign = 'nosign' if sign == '' else  sign        remark = Member['RemarkName']        remark = 'noremark' if remark == '' else remark        alias = Member['Alias']        alias = 'noalias' if alias == '' else alias        nick = Member['NickName']        nick = 'nonick' if nick == '' else nickprint(name, '|||', city, '|||', Member['Sex'], '|||', Member['StarFriend'], '|||', sign,'|||', remark, '|||', alias, '|||', nick)

if __name__ == '__main__':    main()    print('回车键退出...')    input()

import refrom selenium import webdriverfrom selenium.common.exceptions import TimeoutExceptionfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.support.ui import WebDriverWaitfrom selenium.webdriver.support import expected_conditions as ECfrom pyquery import PyQuery as pqfrom taobao.config import *import pymongo

client = pymongo.MongoClient(MONGO_URL)db = client[MONGO_DB]

browser = webdriver.PhantomJS(service_args=SERVICE_ARGS)wait = WebDriverWait(browser, 10)#设置窗口大小，以免默认的相对较小的窗口影响操作browser.set_window_size(1400, 900)

def search():    print('正在搜索')    try:        browser.get('https://www.taobao.com')        # http://selenium-python.readthedocs.io/waits.html#explicit-waits        # 输入框        input = wait.until(            EC.presence_of_element_located((By.CSS_SELECTOR, '#q'))        )        #搜索提交按钮        submit = wait.until(            EC.element_to_be_clickable((By.CSS_SELECTOR, '#J_TSearchForm > div.search-button > button')))

        input.send_keys(KEYWORD)        submit.click()

        #等待搜索内容加载，表示为共x页        total = wait.until(            EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.total')))        #total中就是总共有多少页        get_products()        return total.textexcept TimeoutException:        # wait在网速过慢的时候会出现超时错误，所以我们递归调用重新请求        return search()

def next_page(page_number):    #进入下一页有多种方式，比如点击xx页，比如点击下一页按钮    #比如在输入框中设置页码，点击确定    #我们采用最后一种，因为第一章容易错乱    #而最后一种相比于第二种方便我们在出错的时候递归    print('正在翻页', page_number)    try:        #输入页码框        input = wait.until(            EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.form > input'))        )        #提交按钮        submit = wait.until(EC.element_to_be_clickable(            (By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.form > span.btn.J_Submit')))        #清楚原先的页码        input.clear()        input.send_keys(page_number)        submit.click()        #判断翻页是否成功（判断条件是高亮的当前代码）        wait.until(EC.text_to_be_present_in_element(            (By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > ul > li.item.active > span'), str(page_number)))        get_products()    except TimeoutException:        #出错则递归，重新执行        next_page(page_number)

#解析def get_products():    #判断所有的宝贝信息是否传递成功    wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-itemlist .items .item')))    html = browser.page_source    doc = pq(html)    items = doc('#mainsrp-itemlist .items .item').items()    for item in items:        product = {            'image': item.find('.pic .img').attr('src'),'price': item.find('.price').text(),'deal': item.find('.deal-cnt').text()[:-3],'title': item.find('.title').text(),'shop': item.find('.shop').text(),'location': item.find('.location').text()        }        print(product)        save_to_mongo(product)

#保存def save_to_mongo(result):    try:        if db[MONGO_TABLE].insert(result):            print('存储到MONGODB成功', result)    except Exception:        print('存储到MONGODB失败', result)

def main():    try:        total = search()        #提取总页数        total = int(re.compile('(\d+)').search(total).group(1))        for i in range(2, total + 1):            #从第二页才开始需要点击下一页            next_page(i)    except Exception:        print('出错啦')    finally:        browser.close()

if __name__ == '__main__':    main()

MONGO_URL = 'localhost'MONGO_DB = 'taobao'MONGO_TABLE = 'product'

# http://phantomjs.org/api/command-line.html#第一个参数是不加载图片，使程序运行更快#第二个的开启缓存SERVICE_ARGS = ['--load-images=false', '--disk-cache=true']

KEYWORD = '汉服'

import jsonimport osfrom urllib.parse import urlencodeimport pymongoimport requestsfrom bs4 import BeautifulSoupfrom requests.exceptions import ConnectionErrorimport refrom multiprocessing import Poolfrom hashlib import md5from json.decoder import JSONDecodeErrorfrom spider_basic.config import *

#client = pymongo.MongoClient(MONGO_URL, connect=False)db = client[MONGO_DB]

#获取主页，offse是ajax动态加载的偏移量，keyword是搜索的关键字def get_page_index(offset, keyword):    data = {        'autoload': 'true','count': 20,'cur_tab': 3,'format': 'json','keyword': keyword,'offset': offset,}    params = urlencode(data)    base = 'http://www.toutiao.com/search_content/'    url = base + '?' + paramstry:        response = requests.get(url)        if response.status_code == 200:            return response.textreturn None    except ConnectionError:        print('Error occurred')        return None

#下载def download_image(url):    print('Downloading', url)    try:        response = requests.get(url)        if response.status_code == 200:            save_image(response.content)        return None    except ConnectionError:        return None

#保存def save_image(content):    # 文件名    # 路径 名称 后缀    # 名称md5 这是为了当我们运行一次程序出错时，第二次不再保存重复图片    file_path = '{0}/{1}/{2}.{3}'.format(os.getcwd(),'pic',md5(content).hexdigest(), 'jpg')    print(file_path)    # 如果文件不存在则保存    # content是二进制形式的网页内容    if not os.path.exists(file_path):        with open(file_path, 'wb') as f:            f.write(content)            f.close()

#解析搜索后返回的网页（就是图集XHR中的那些）def parse_page_index(text):    try:        #转化为JSON        data = json.loads(text)        #如果JSON数据中含有data这个键        if data and 'data' in data.keys():            for item in data.get('data'):                yield item.get('article_url')    except JSONDecodeError:        pass

#获取详情页def get_page_detail(url):    try:        response = requests.get(url)        if response.status_code == 200:            return response.textreturn None    except ConnectionError:        print('Error occurred')        return None

#解析详情页def parse_page_detail(html, url):    soup = BeautifulSoup(html, 'lxml')    #获取标题    result = soup.select('title')    title = result[0].get_text() if result else ''    #获取图片地址（定义正则规则，并search）    images_pattern = re.compile('var gallery = (.*?);', re.S)    result = re.search(images_pattern, html)    #如果非空    if result:        #转化为JSON格式        data = json.loads(result.group(1))        #如果sub_images这个键的值对应了各种url        if data and 'sub_images' in data.keys():            sub_images = data.get('sub_images')            images = [item.get('url') for item in sub_images]            #下载图片            for image in images: download_image(image)            return {                'title': title,'url': url,'images': images}

#存储到数据库def save_to_mongo(result):    if db[MONGO_TABLE].insert(result):        print('Successfully Saved to Mongo', result)        return True    return False

def main(offset):    text = get_page_index(offset, KEYWORD)    #生成器返回图集中每一个图集的URL的遍历器    urls = parse_page_index(text)    #遍历每一个图集，获取详情页信息    for url in urls:        #获取详情页        html = get_page_detail(url)        #解析详情页        result = parse_page_detail(html, url)        #保存到MongoDB        if result: save_to_mongo(result)

# if __name__ == '__main__':#     main(60)

# 注意，如果不在if __name__ == '__main__':中运行的话，会爆出一堆多线程的错误if __name__ == '__main__':    pool = Pool()    groups = ([x * 20 for x in range(GROUP_START, GROUP_END + 1)])    pool.map(main, groups)    pool.close()    pool.join()

MONGO_URL = 'localhost'MONGO_DB = 'toutiao'MONGO_TABLE = 'toutiao'

GROUP_START = 1GROUP_END = 20KEYWORD='萌宠'

附件列表