python爬虫淘宝实例-python 淘宝爬虫示例源码（抓取天猫数据）

【实例简介】爬取淘宝天猫网站数据

【实例截图】

【核心代码】

# -*- coding: utf-8 -*-

#!/usr/bin/env python

import datetime

import urlparse

import socket

import scrapy

from scrapy.loader.processors import MapCompose, Join

from scrapy.loader import ItemLoader

from scrapy.http import Request

import json

import base64

import scrapy

from scrapy.http.headers import Headers

from taobao.items import TaobaoItem

from urllib import quote,unquote

import sys

reload(sys)

sys.setdefaultencoding('utf-8')

class MySpider(scrapy.Spider):

name = 'tmall2'

start_urls = ["http://example.com", "http://example.com/foo"]

def __init__(self):

self.headers={

'Host': 'detail.tmall.com',

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:44.0) Gecko/20100101 Firefox/44.0',

'Accept': 'text/html,application/xhtml xml,application/xml;q=0.9,*/*;q=0.8',

'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',

'Accept-Encoding':'gzip, deflate, br',

'Referer':'https://list.tmall.com/search_product.htm?q=iphone',

'Cookie':'hng=CN%7Czh-cn%7CCNY; l=AmFhUQz9l9Bm0s1PIcUbVzUrUSd709Vr; pnm_cku822=213UW5TcyMNYQwiAiwTR3tCf0J%2FQnhEcUpkMmQ%3D%7CUm5OcktzSHFMdkpwTXFEcSc%3D%7CU2xMHDJ%2BH2QJZwBxX39RaFF%2FX3EtTCpGIV8lC10L%7CVGhXd1llXGRfZlthXWdaZlNmUWxOdEpxRXhMeUx0QHhMckh2Qmw6%7CVWldfS0SMg4zBycbJAQqAXRfeB9kNFY0EDtEajxq%7CVmhIGCwSMg8vEycaJAQ6DzQIKBQgHyICPgM2CysXIxwhAT0AOQRSBA%3D%3D%7CV25Tbk5zU2xMcEl1VWtTaUlwJg%3D%3D; cna=c7xUD5TeoxgCARsmEAVdwH4E; cq=ccp%3D1; t=ea7cda7b4dd7d94c574c51a61cd68bf6; uc3=nk2=G4mgLCRZx6no8qfi5g%3D%3D&id2=UonZBtTqYSCQGg%3D%3D&vt3=F8dAScn1mkMKfq3pmos%3D&lg2=W5iHLLyFOGW7aA%3D%3D; lgc=xiaowenjie886; tracknick=xiaowenjie886; _tb_token_=WcXcAjsXNiib; cookie2=3647140634e8134de4621d27d06a6239; OZ_1U_2061=vid=v6cf00b635ac22.0&ctime=1456406710&ltime=0; OZ_1Y_2061=erefer=https%3A//list.tmall.com/search_product.htm%3Fq%3D%25CD%25E2%25CC%25D7%25C4%25D0%26click_id%3D%25CD%25E2%25CC%25D7%25C4%25D0%26from%3Dmallfp..pc_1.0_hq%26spm%3D875.7789098.a1z5h.1.1DJapJ&eurl=https%3A//detail.tmall.com/item.htm%3Fspm%3Da220m.1000858.1000725.11.XG2djx%26id%3D525068649325%26skuId%3D3125134725161%26areaId%3D440300%26cat_id%3D50025174%26rn%3D020410dd2019f68eaf3d848b4d14552f%26user_id%3D196993935%26is_b%3D1&etime=1456406710&ctime=1456406710&ltime=0&compid=2061',

'Connection':'keep-alive',

'Cache-Control':'max-age=0'

}

self.cookies={

'l':'ArGxZLdew/Qq2hKqnZPLZoKK4TdLHyUb',

'cna':'OW9VD5ReU2ACAdxw7hJSgV4y',

'cookie2':'1cfecc6ae5749b36804d524b9d0cccb4',

't':'2fd2137e54b753c57bec7b945f504547',

'_tb_token_':'l0ckiPAV9KXX',

'ck1':'',

'uc1':'cookie14=UoWyiPlLPWymJA%3D%3D&existShop=false&cookie16=U%2BGCWk%2F74Mx5tgzv3dWpnhjPaQ%3D%3D&cookie21=WqG3DMC9EdFmJgke4t0pDw%3D%3D&tag=3&cookie15=VT5L2FSpMGV7TQ%3D%3D&pas=0',

'uc3':'nk2=G4mgLCRZx6no8qfi5g%3D%3D&id2=UonZBtTqYSCQGg%3D%3D&vt3=F8dAScn1nphE%2FG5b7yQ%3D&lg2=Vq8l%2BKCLz3%2F65A%3D%3D',

'lgc':'xiaowenjie886',

'tracknick':'xiaowenjie886',

'cookie1':'UNaG7hUVmBqzT5U4J5xH8HeBiBsUUL0QGHEE%2BJc503Q%3D',

'unb':'1821174258',

'skt':'116663449cdcca0c',

'_nk_':'xiaowenjie886',

'_l_g_':'Ug%3D%3D',

'cookie17':'UonZBtTqYSCQGg%3D%3D',

'hng':'CN%7Czh-cn%7CCNY',

'login':'true',

'pnm_cku822':'pnm_cku822=213UW5TcyMNYQwiAiwTR3tCf0J%2FQnhEcUpkMmQ%3D%7CUm5OcktzSHFMdkpwTXFEcSc%3D%7CU2xMHDJ%2BH2QJZwBxX39RaFF%2FX3EtTCpGIV8lC10L%7CVGhXd1llXGRfZlthXWdaZlNmUWxOdEpxRXhMeUx0QHhMckh2Qmw6%7CVWldfS0SMg4zBycbJAQqAXRfeB9kNFY0EDtEajxq%7CVmhIGCwSMg8vEycaJAQ6DzQIKBQgHyICPgM2CysXIxwhAT0AOQRSBA%3D%3D%7CV25Tbk5zU2xMcEl1VWtTaUlwJg%3D%3D; expires=Sat, 26 Mar 2016 13:32:50 GMT; path=/; domain=detail.tmall.com'

}

self.url='https://s.taobao.com/search?spm=a21bo.7724922.8452-fline.1.uFDF4G&q=秋季打底衫'

def start_requests(self):

script="""

function main(splash)

assert(splash:go(splash.args.url))

splash:wait(1.0)

return splash:html()

end

"""

yield scrapy.Request(self.url,self.parse_result, meta={

'splash': {

'args': {'lua_source': script,'url':self.url},

'endpoint': 'execute',

}

})

def parse_result(self, response):

pageCountXpath=response.xpath("//div[@class='pager']/ul/li[2]/text()").extract()

page=(','.join(pageCountXpath))[1:]

pagecount=int(page)

script="""

function main(splash)

assert(splash:go(splash.args.url))

assert(splash:wait(8.5))

return splash:html()

end

"""

for i in range(0,44*pagecount,44):

url2='https://s.taobao.com/search?q=秋季打底衫&s=%d' % i

yield scrapy.Request(url2,self.parse_next,meta={

'splash':{

'args':{'lua_source':script,'url':url2},

'endpoint':'execute',

}

})

def parse_next(self,response):

item = TaobaoItem()

titleALL=response.xpath("//div[@class='item ']/div[2]/div[2]/a/text()").extract()

item['title']=titleALL

shopnameAll =response.xpath("//a[@class='shopname J_MouseEneterLeave J_ShopInfo']/span[2]/text()").extract()

item["shopname"]=shopnameAll

return item

#return item

# sudo service docker restart

python爬虫淘宝实例-python 淘宝爬虫示例源码（抓取天猫数据）相关推荐

python爬虫代码实例源码_python 淘宝爬虫示例源码（抓取天猫数据）
爬取淘宝天猫网站数据# -*- coding: utf-8 -*- #!/usr/bin/env Python import dateTime import URLparse import sock ...
python tkinter实例_python绘制一个图形示例源码(tkinter)
[实例简介] [实例截图] [核心代码] # -*- coding: utf-8 -*- #!/usr/bin/python import math from tkinter import * cla ...
python爬虫淘宝实例-Python——爬虫实战爬取淘宝店铺内所有宝贝图片
之前用四篇很啰嗦的入门级别的文章,带着大家一起去了解并学习在编写爬虫的过程中,最基本的几个库的用法. 那么今天,我们就正式开始我们的第一篇实战内容,爬取一整个淘宝店铺里的所有宝贝的详情页,并且把详情页 ...
python爬虫知网实例-python爬虫实例项目大全
WechatSogou [1]- 微信公众号爬虫.基于搜狗微信搜索的微信公众号爬虫接口,可以扩展成基于搜狗搜索的爬虫,返回结果是列表,每一项均是公众号具体信息字典. DouBanSpider [2]- ...
实现从淘宝（天猫）定时抓取订单数据、打印电子面单并保存到ERP表中
实现从淘宝(天猫)定时抓取订单数据.打印电子面单并保存到ERP表中前言实现思路代码片段参考前言最近有厂商提出想把天猫店铺的数据拿到后台ERP管理系统中,并能实现线下打印电子面单功能.接手这个 ...
淘宝主图视频怎么上传？怎么抓取、下载？
淘宝.阿里上的主图视频,很多商家都不会制作主图视频,而又眼馋主图视频来带的流量,而在淘宝上的商家很多都是在阿里进货的,代发.代销的商家,在商品的主图和视频上是比较缺乏资源的,所以商家如果想要获取主图视 ...
淘宝、阿里的商品主图视频如何抓取、下载的步骤
阿里平台的商品主图视频如何下载.抓取.保存上传到淘宝平台呢?这个是淘宝.天猫代发.代销商家目前比较烦心的事了,在电商平台上开店的商家很多都不是专业的卖家,很多商家都是刚刚接触电商平台.开店的毛头小子, ...
Python爬虫抓取动态数据
一个月前实习导师布置任务说通过网络爬虫获取深圳市气象局发布的降雨数据,网页如下: 心想,爬虫不太难的,当年跟zjb爬煎蛋网无(mei)聊(zi)图的时候,多么清高.由于接受任务后的一个月考试加作业一大 ...
python推特爬虫_Tweepy1_抓取Twitter数据
之前一直想用爬虫登陆并抓取twitter数据,试过scrapy,requests等包,都没成功,可能是我还不太熟悉的原因,不过今天发现了一个新包tweepy,专门用于在Python中处理twitte ...

python爬虫淘宝实例-python 淘宝爬虫示例源码（抓取天猫数据）

python爬虫淘宝实例-python 淘宝爬虫示例源码（抓取天猫数据）相关推荐

最新文章

热门文章