根据oe抓取ebayno title fits

示例：

网址：https://www.ebay.com/sch/am-autoparts/m.html?item=371393241499&rt=nc&_trksid=p2047675.l4064

oe:171340L

这里需要设置美国收货地址，否则搜索条数少

代码如下：

import logging
import random
import threading
import requests
import pandas as pd
from queue import Queue
import pymysql
from bs4 import BeautifulSoup
from http.cookiejar import CookieJar
import time
import  re
import csv
import json
import os
class Spider():def __init__(self):self.SESSION = requests.session()self.SESSION.cookies = CookieJar()# print(self.SESSION.cookies)self.HEAD = self.randHeader()self.cookie = self.set_shipping()def randHeader(self):head_connection = ['Keep-Alive', 'close']head_accept = ['text/html, application/xhtml+xml, */*']head_accept_language = ['zh-CN,fr-FR;q=0.5', 'en-US,en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3']head_user_agent = ['Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko','Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36','Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; rv:11.0) like Gecko)','Mozilla/5.0 (Windows; U; Windows NT 5.2) Gecko/2008070208 Firefox/3.0.1','Mozilla/5.0 (Windows; U; Windows NT 5.1) Gecko/20070309 Firefox/2.0.0.3','Mozilla/5.0 (Windows; U; Windows NT 5.1) Gecko/20070803 Firefox/1.5.0.12','Opera/9.27 (Windows NT 5.2; U; zh-cn)','Mozilla/5.0 (Macintosh; PPC Mac OS X; U; en) Opera 8.0','Opera/8.0 (Macintosh; PPC Mac OS X; U; en)','Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.12) Gecko/20080219 Firefox/2.0.0.12 Navigator/9.0.0.6','Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Win64; x64; Trident/4.0)','Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)','Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Maxthon/4.0.6.2000 Chrome/26.0.1410.43 Safari/537.1 ','Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E; QQBrowser/7.3.9825.400)','Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0 ','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.92 Safari/537.1 LBBROWSER','Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; BIDUBrowser 2.x)','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/3.0 Safari/536.11']header = {'Connection': head_connection[0],'Accept': head_accept[0],'Accept-Language': head_accept_language[1],'User-Agent': head_user_agent[random.randrange(0, len(head_user_agent))]}return headerdef getBeautifulSoup(self, query_rl):r = self.SESSION.get(url=query_rl, headers=self.HEAD)soup = BeautifulSoup(r.text, 'html.parser')return soupdef set_shipping(self):serchWords = "2000 Chevy silverado window regulator".replace(" ", "%20")query_rl = "https://www.ebay.com/sch/i.html?_from=R40&_sacat=0&_ipg=20&rt=nc&_nkw=" + serchWords + "&_pgn=1&_skc=0"r = self.SESSION.get(url=query_rl, headers=self.HEAD)soup = BeautifulSoup(r.text, 'html.parser')# 获取第一个ebayno，将收货地址更改itm = soup.find("div", "lvpic pic img left")['iid']getrates_url = "http://www.ebay.com/itm/getrates?item=" + itm + "&country=1&co=0&cb=jQuery1705349737076189762_1501724760425"r = self.SESSION.get(url=getrates_url, headers=self.HEAD)  # 发请求，保存cookiedef getfits(self,oe):oe = str(oe)print(oe)# self.set_shipping()url = 'https://www.ebay.com/sch/m.html?_odkw=&_ssn=am-autoparts&item=371393241499&_osacat=0&_from=R40&_trksid=p2046732.m570.l1313.TR0.TRC0.H0.X' + oe + '.TRS0&_nkw=' + oe + '&_sacat=0'soup = self.getBeautifulSoup(url)content = soup.find_all(attrs={'class': re.compile('^sresult lvresult clearfix li')}) #找到每个商品result = []for c in content:ebayno = c.find("div", "lvpic pic img left")["iid"] #定位ebayno# title_href =  c.find("a", class_="vip")["href"]title = c.find("a", class_="vip").string #定位titleurl = "http://vi.vipr.ebaydesc.com/ws/eBayISAPI.dll?ViewItemDescV4&item=" + ebayno + "&t=1505325002000&tid=10&category=33590&seller=am-autoparts&vipguid=5afca32015e0a9945a40d25cff951777&excTrk=1&tto=3500&lsite=100&ittenable=false&domain=ebay.com&descgauge=1&cspheader=1"sub_page_soup = self.getBeautifulSoup(url)fits = sub_page_soup.find("div", class_="item_fits_con") #定位fitsif fits is None:result.append([oe, ebayno, title, "#"])continueflis = fits.find_all("li")for fli in flis:fitname = fli.string# print(fitname)result.append([oe, ebayno, title,fitname])df = pd.DataFrame(result,columns=["oe","ebayno","title","fits"])df.to_csv("fits/fits_"+oe + ".csv",index=False)class ThreadCrawl(threading.Thread): #ThreadCrawl类继承了Threading.Thread类def __init__(self, queue):  #子类特有属性， queueFORMAT = time.strftime("[%Y-%m-%d %H:%M:%S]", time.localtime()) + "[Spider]-----%(message)s------"logging.basicConfig(level=logging.INFO, format=FORMAT)threading.Thread.__init__(self)self.queue = queueself.spider = Spider()  #子类特有属性spider， 并初始化，将实例用作属性def run(self):while True:success = Trueitem = self.queue.get() #调用队列对象的get()方法从队头删除并返回一个项目itemself.spider.getfits(item)# try:#     self.spider.getfits(item) #调用实例spider的方法getDataById(item)# except :#     success = False# if not success :#     self.queue.put(item)logging.info("now queue size is: %d" % self.queue.qsize()) #队列对象qsize()方法，返回队列的大小self.queue.task_done() #队列对象在完成一项工作后，向任务已经完成的队列发送一个信号class SpiderJob():def __init__(self , size , qs):self.size = size  # 将形参size的值存储到属性变量size中self.qs = qsdef work(self):toSpiderQueue = Queue() #创建一个Queue队列对象for i in range(self.size):t = ThreadCrawl(toSpiderQueue)    #将实例用到一个类的方法中t.setDaemon(True)t.start()for q in self.qs:toSpiderQueue.put(q)  #调用队列对象的put()方法，在对尾插入一个项目itemtoSpiderQueue.join()    #队列对象，等到队列为空，再执行别的操作

代码二：

from  fits_ebay import SpiderJob #从一个模块中导入类
import logging
import random
import threading
import requests
import pandas as pd
from queue import Queue
import pymysql
from bs4 import BeautifulSoup
from http.cookiejar import CookieJar
import time
import  re
import csv
import json
import osif __name__ == '__main__':qs = []with open("oe.txt") as fp:for line in fp:qs.append(str(line.strip()))# print(qs)## #第一次跑的时候应该注释# no_int = []# with open("oe.txt") as fp:#     for line in fp:#         ebayno = line.strip()#         if os.path.exists("fits\\fits_" + ebayno + ".csv"):  # 判断是否存在文件#             continue#         else:#             no_int.append(ebayno)# print(no_int)# qs = no_intprint(qs)Job = SpiderJob(8, qs)Job.work()

根据oe抓取ebayno title fits相关推荐

python爬虫实战（四）：selenium爬虫抓取阿里巴巴采购批发商品
一.前言本编教程是通过selenium爬取阿里巴巴采购批发网址的商品数据信息,并保存到mongo.为什么用selenium呢,是因为网站大多数不再是静态网站,而是有诸如Ajax请求的动态网站,如果接 ...
怎么用PHP抓取百度排名？
PHP抓取百度排名代码: [PHP]代码 function s($keyword,$url,$page = 1){ static $px = 0; $rsState = false; $enKeywo ...
php爬虫抓取信息及反爬虫相关
php爬虫首推Curl函数了,先来认识下它. 0x01.curl扩展的安装: 1.确保php子文件夹ext里面有php_curl.dll(一般都有的,一般配置时候会设置环境变量的) 2.将php.in ...
批量抓取title keywords descrip【seo工具】
前两天帮朋友写个小工具.思路很简单实现也不是太难.写了这么个小工具. 实现功能:通过搜索引擎自动换页抓取记录页面中的属性值.贴代码: using System; using System.Collec ...
python 图片抓取
写个爬虫,抓点百度图片.本程序基于python3编写,可根据关键词抓取百度图片下载的图片: 代码: import json import os import random import time i ...
大众点评网地点抓取程序php
<?php set_time_limit(0); class snap_dianping { //抓取的分类,(*)代表页码 static $url = array( '川菜'=>'htt ...
jupyter分割代码块_科研分享—Python根据关键词自动抓取Pubmed文献标题（附全部代码）文末有福利...
写在前面:接触Python应该是8月初的一篇公众号文章,大致内容是使用py爬取数据库并汇总到本地.正好手头需要对某个领域的文献进行调研,不妨学习一下. 什么是Python? 百度说:Python (计 ...
.NET2.0抓取网页全部链接【月儿原创】
.NET2.0抓取网页全部链接作者:清清月儿主页:http://blog.csdn.net/21aspnet/ 时间:2007.4.18 该方法经过对各大门户网站测试结果是抓取 ...
构建node.js基础镜像_我如何使用Node.js构建工作抓取网络应用
构建node.js基础镜像 by Oyetoke Tobi Emmanuel 由Oyetoke Tobi Emmanuel 我如何使用Node.js构建工作抓取网络应用 (How I built a ...

根据oe抓取ebayno title fits

根据oe抓取ebayno title fits相关推荐

最新文章

热门文章