python爬虫代码实例源码_python 淘宝爬虫示例源码（抓取天猫数据）

爬取淘宝天猫网站数据# -*- coding: utf-8 -*-

#!/usr/bin/env Python

import dateTime

import URLparse

import socket

import scrapy

from scrapy.loader.processors import MapCompose, Join

from scrapy.loader import ItemLoader

from scrapy.http import Request

import json

import base64

import scrapy

from scrapy.http.headers import Headers

from taobao.items import TaobaoItem

from urllib import quote,unquote

import sys

reload(sys)

sys.setDEFAULTencoding('utf-8')

class MySpider(scrapy.Spider):

name = 'tmall2'

start_urls = ["http://example.com", "http://example.com/foo"]

def __init__(self):

self.headers={

'Host': 'detail.tmall.com',

'user-Agent': 'Mozilla/5.0 (windows NT 10.0; WOW64; rv:44.0) GECko/20100101 Firefox/44.0',

'Accept': 'text/html,application/xhtml xml,application/xml;q=0.9,*/*;q=0.8',

'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',

'Accept-Encoding':'gzip, deflate, br',

'Referer':'https://list.tmall.com/search_product.htm?q=iphone',

'Cookie':'hng=CN%7Czh-cn%7CCNY; l=AmFhUQz9l9Bm0s1PIcUbVzUrUSd709Vr; pnm_cku822=213UW5TcyMNYQwiAiwTR3tCf0J%2FQnhEcUpkMmQ%3D%7CUm5OcktzSHFMdkpwTXFEcSc%3D%7CU2xMHDJ%2BH2QJZwBxX39RaFF%2FX3EtTCpGIV8lC10L%7CVGhXd1llXGRfZlthXWdaZlNmUWxOdEpxRXhMeUx0QHhMckh2Qmw6%7CVWldfS0SMg4zBycbJAQqAXRfeB9kNFY0EDtEajxq%7CVmhIGCwSMg8vEycaJAQ6DzQIKBQgHyICPgM2CysXIxwhAT0AOQRSBA%3D%3D%7CV25Tbk5zU2xMcEl1VWtTaUlwJg%3D%3D; cna=c7xUD5TeoxgCARsmEAVdwH4E; cq=ccp%3D1; t=ea7cda7b4dd7d94c574c51a61cd68bf6; uc3=nk2=G4mgLCRZx6no8qfi5g%3D%3D&id2=UonZBtTqYSCQGg%3D%3D&vt3=F8dAscn1mkMKfq3pmos%3D&lg2=W5iHLLyFOGW7aA%3D%3D; lgc=xiaowenjie886; tracknick=xiaowenjie886; _tb_token_=WcXcAjsXNiib; cookie2=3647140634e8134de4621d27d06a6239; OZ_1U_2061=vid=v6cf00b635ac22.0&ctime=1456406710&ltime=0; OZ_1Y_2061=erefer=https%3A//list.tmall.com/search_product.htm%3Fq%3D%25CD%25E2%25CC%25D7%25C4%25D0%26click_id%3D%25CD%25E2%25CC%25D7%25C4%25D0%26from%3Dmallfp..pc_1.0_hq%26spm%3D875.7789098.a1z5h.1.1DJapJ&eurl=https%3A//detail.tmall.com/item.htm%3Fspm%3Da220m.1000858.1000725.11.XG2djx%26id%3D525068649325%26skuId%3D3125134725161%26areaId%3D440300%26cat_id%3D50025174%26rn%3D020410dd2019f68eaf3d848b4d14552f%26user_id%3D196993935%26is_b%3D1&etime=1456406710&ctime=1456406710&ltime=0&compid=2061',

'Connection':'keep-alive',

'cache-Control':'max-age=0'

}

self.cookies={

'l':'ArGxZLdew/Qq2hKqnZPLZoKK4TdLHyUb',

'cna':'OW9VD5ReU2Acadxw7hJSgV4y',

'cookie2':'1cfecc6ae5749b36804d524b9d0cccb4',

't':'2fd2137e54b753c57bec7b945f504547',

'_tb_token_':'l0ckiPAV9KXX',

'ck1':'',

'uc1':'cookie14=UoWyiPlLPWymJA%3D%3D&existShop=false&cookie16=U%2BGCWk%2F74Mx5tgzv3dWpnhjPaQ%3D%3D&cookie21=WqG3DMC9EdFmJgke4t0pDw%3D%3D&tag=3&cookie15=VT5L2FSpMGV7TQ%3D%3D&pas=0',

'uc3':'nk2=G4mgLCRZx6no8qfi5g%3D%3D&id2=UonZBtTqYSCQGg%3D%3D&vt3=F8dAScn1nphE%2FG5b7yQ%3D&lg2=Vq8l%2BKCLz3%2F65A%3D%3D',

'lgc':'xiaowenjie886',

'tracknick':'xiaowenjie886',

'cookie1':'UNaG7hUVmBqzT5U4J5xH8HeBiBsUUL0QGHEE%2BJc503Q%3D',

'unb':'1821174258',

'skt':'116663449cdcca0c',

'_nk_':'xiaowenjie886',

'_l_g_':'Ug%3D%3D',

'cookie17':'UonZBtTqYSCQGg%3D%3D',

'hng':'CN%7Czh-cn%7CCNY',

'login':'true',

'pnm_cku822':'pnm_cku822=213UW5TcyMNYQwiAiwTR3tCf0J%2FQnhEcUpkMmQ%3D%7CUm5OcktzSHFMdkpwTXFEcSc%3D%7CU2xMHDJ%2BH2QJZwBxX39RaFF%2FX3EtTCpGIV8lC10L%7CVGhXd1llXGRfZlthXWdaZlNmUWxOdEpxRXhMeUx0QHhMckh2Qmw6%7CVWldfS0SMg4zBycbJAQqAXRfeB9kNFY0EDtEajxq%7CVmhIGCwSMg8vEycaJAQ6DzQIKBQgHyICPgM2CysXIxwhAT0AOQRSBA%3D%3D%7CV25Tbk5zU2xMcEl1VWtTaUlwJg%3D%3D; expires=Sat, 26 Mar 2016 13:32:50 GMT; path=/; domain=detail.tmall.com'

}

self.url='https://s.taobao.com/search?spm=a21bo.7724922.8452-fline.1.uFDF4G&q=秋季打底衫'

def start_requests(self):

script="""

function main(splash)

assert(splash:go(splash.args.url))

splash:wait(1.0)

return splash:html()

end

"""

yield scrapy.Request(self.url,self.parse_result, Meta={

'splash': {

'args': {'lua_source': script,'url':self.url},

'endpoint': 'execute',

}

})

def parse_result(self, response):

pageCountXpath=response.xpath("//div[@class='pager']/ul/li[2]/text()").extract()

page=(','.join(pageCountXpath))[1:]

pagecount=int(page)

script="""

function main(splash)

assert(splash:go(splash.args.url))

assert(splash:wait(8.5))

return splash:html()

end

"""

for i in range(0,44*pagecount,44):

url2='https://s.taobao.com/search?q=秋季打底衫&s=%d' % i

yield scrapy.Request(url2,self.parse_next,meta={

'splash':{

'args':{'lua_source':script,'url':url2},

'endpoint':'execute',

}

})

def parse_next(self,response):

item = TaobaoItem()

titleALL=response.xpath("//div[@class='item ']/div[2]/div[2]/a/text()").extract()

item['title']=titleALL

shopnameAll =response.xpath("//a[@class='shopname J_MouseEneterLeave J_ShopInfo']/span[2]/text()").extract()

item["shopname"]=shopnameAll

return item

#return item

# sudo service docker restart

python爬虫代码实例源码_python 淘宝爬虫示例源码（抓取天猫数据）相关推荐

python获取app信息的库_Python学习教程：另辟蹊径，appium抓取app应用数据了解一下...
作为爬虫工程师,没有价格不知道selenium的. 什么是selenium? Selenium原本是一个用于Web应用程序自动化测试工具.Selenium测试直接运行在浏览器中,就像真正的用户在操作一 ...
python爬虫淘宝实例-python 淘宝爬虫示例源码（抓取天猫数据）
[实例简介]爬取淘宝天猫网站数据 [实例截图] [核心代码] # -*- coding: utf-8 -*- #!/usr/bin/env python import datetime import ...
python淘宝爬虫基于requests抓取淘宝商品数据_python淘宝爬虫基于requests抓取淘宝商品数据...
在学校蹭过python的课,觉得python异常强大,趁寒假有时间,瞎搞一下,希望能和大伙一起探讨学习.废话不多说了,直接正题. requests 是Python的http库,可以完成绝大部分与htt ...
代购源码，淘宝代购系统源码，代购程序，代购系统源码PHP前端源码参数说明
> 代购业务场景: 代购业务近年兴起的一种购物模式,是帮国外客户购买中国商品.主要通过外贸代购模式,把淘宝. 天猫等电商平台的全站商品通过API接入到你的网站上,瞬间就可以架设一个有数亿产品的大 ...
获取淘宝商品分类详情API，抓取淘宝全品类目API接口分享(代码展示、参数说明）
商品分类技巧淘宝店铺分类怎么设置?我们登录卖家账号的时候,我们看到自己的商品,会想要给商品进行分类,一个好的分类可以帮助提高商品的曝光率.那么在给商品分类前,如果您毫无头绪,以下几点可以给您带来一点 ...
淘宝页面商品信息的抓取
文章目录概述登录淘宝商品搜索页面的访问应对反爬虫的一些尝试概述使用selenium登录淘宝并抓取关键字"iPad"对应页面的商品信息. 页面的抓取使用requests应 ...
python简单爬虫手机号_python手机号前7位归属地爬虫代码实例
需求分析项目上需要用到手机号前7位,判断号码是否合法,还有归属地查询.旧的数据是几年前了太久了,打算用python爬虫重新爬一份单线程版本 # coding:utf-8 import reques ...
python开源爬虫项目违法吗_Python开源爬虫项目代码：抓取淘宝、京东、QQ、知网数据...
数据挖掘入门与实战公众号: datadw scrapy_jingdong[9]- 京东爬虫.基于scrapy的京东网站爬虫,保存格式为csv.[9]: https://github.com/taiz ...
23个Python爬虫开源项目代码，包含微信、淘宝、豆瓣、知乎、微博等
今天为大家整理了23个Python爬虫项目.整理的原因是,爬虫入门简单快速,也非常适合新入门的小伙伴培养信心,所有链接指向GitHub,微信不能直接打开,老规矩,可以用电脑打开. 关注公众号「Pyth ...

python爬虫代码实例源码_python 淘宝爬虫示例源码（抓取天猫数据）

python爬虫代码实例源码_python 淘宝爬虫示例源码（抓取天猫数据）相关推荐

最新文章

热门文章