世纪佳缘信息爬取存储到mysql,下载图片到本地，从数据库选取账号对其发送消息更新发信状态

利用这种方法，可以把所有会员信息存储下来，多线程发信息，10秒钟就可以对几百个会员完成发信了。

首先是筛选信息后爬取账号信息，

#-*-coding:utf-8-*-
import requests,re,json,time,threadpool,os
from mydba import MySql
from gevent import monkey#monkey.patch_all()

header={'Cookie':'guider_quick_search=on; SESSION_HASH=xxxxxxxxcd4523713c3d3700579350daa9992f1a; user_access=1; save_jy_login_name=1314xxxx; sl_jumper=%26cou%3D17; last_login_time=1498576743; user_attr=000000; pclog=%7B%22160961843%22%3A%221498576757268%7C1%7C0%22%7D; IM_S=%7B%22IM_CID%22%3A8677484%2C%22svc%22%3A%7B%22code%22%3A0%2C%22nps%22%3A0%2C%22unread_count%22%3A%220%22%2C%22ocu%22%3A0%2C%22ppc%22%3A0%2C%22jpc%22%3A0%2C%22regt%22%3A%221486465367%22%2C%22using%22%3A%2240%2C33%2C2%2C%22%2C%22user_type%22%3A%2210%22%2C%22uid%22%3A160961843%7D%2C%22IM_SV%22%3A%22123.59.161.3%22%2C%22m%22%3A26%2C%22f%22%3A0%2C%22omc%22%3A0%7D; FROM_BD_WD=%25E4%25B8%2596%25E7%25BA%25AA%25E4%25BD%25B3%25E7%25BC%2598; FROM_ST_ID=416640; FROM_ST=.jiayuan.com; REG_ST_ID=15; REG_ST_URL=http://bzclk.baidu.com/adrc.php?t=06KL00c00f7t0wC0Gfum0QkHAsjnX7Fu00000PNeYH300000uybcI1.THL2sQ1PEPZRVfK85yF9pywd0ZnqryRkryfdryDsnj0kuWR3u0Kd5RuKP17Knj97rDDvPRNAPH9APbRkwHc4njIAnDPjP1NA0ADqI1YhUyPGujYzrH0dnWTYnHckFMKzUvwGujYkP6K-5y9YIZ0lQzqzuyT8ph-9XgN9UB4WUvYETLfE5v-b5HfkPWmYnaudThsqpZwYTjCEQLILIz4Jpy74Iy78QhPEUfKWThnqnWRdnjn&tpl=tpl_10762_15668_1&l=1053916117&attach=location%3D%26linkName%3D%25E6%25A0%2587%25E9%25A2%2598%26linkText%3D%25E4%25B8%2596%25E7%25BA%25AA%25E4%25BD%25B3%25E7%25BC%2598%25E7%25BD%2591%25EF%25BC%2588Jiayuan.com%25EF%25BC%2589%253A2017%25EF%25BC%258C%26xp%3Did(%2522m78828180%2522)%252FDIV%255B1%255D%252FDIV%255B1%255D%252FDIV%255B1%255D%252FH2%255B1%255D%252FA%255B1%255D%26linkType%3D%26checksum%3D191&ie=UTF-8&f=8&tn=baidu&wd=%E4%B8%96%E7%BA%AA%E4%BD%B3%E7%BC%98&oq=%E4%B8%96%E7%BA%AA%E4%BD%B3%E7%BC%98&rqlang=cn; REG_REF_URL=http://www.jiayuan.com/usercp/profile.php?action=work; PHPSESSID=d5557cc5a4560c14c2dc685bcb7fc009; stadate1=159xxxx; myloc=44%7C4403; myage=xx; PROFILE=160xxxxxx%3A%25E9%25A3%258E%3Am%3Aat1.jyimg.com%2F42%2Ffb%2F4558c12f370e446240ba148bcac4%3A1%3A%3A1%3A4558c12f3_2_avatar_p.jpg%3A1%3A1%3A61%3A10; mysex=m; myuid=159xxxxxxx; myincome=40; mylevel=2; main_search:160961843=%7C%7C%7C00; RAW_HASH=J3ewrCGVZG5eU2agrymP2-bNz0IQQpivXOncdsOO63oS3%2AH4o4%2AsAifV0twuNFXqWippm3rMHXvTK%2APmGbap-ZYZpz-18ogWLnBXkcpY87GlFps.; COMMON_HASH=424558c12fxxxxx40ba148bcac4fb; IM_CON=%7B%22IM_TM%22%3A1498577459991%2C%22IM_SN%22%3A3%7D; IM_M=%5B%5D; pop_time=1498577573654; IM_CS=2; IM_ID=12; is_searchv2=1; IM_TK=1498577921417','User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36',
}def fun(i):url='http://search.jiayuan.com/v2/search_v2.php?sex=f&key=&stc=1%3A4403%2C2%3A23.27%2C3%3A155.170%2C23%3A1&sn=default&sv=1&p='+str(i)+'&f=select&listStyle=bigPhoto&pri_uid=xxxx&jsversion=v5'while(1):res=''try:res=requests.get(url,headers=header)except Exception,e:print eif res!='':breakcontent=res.contentdictx= json.loads(re.findall('##jiayser##([\s\S]*?)##jiayser##',content)[0])   ##content是一个json但加了其他东西，需要把json提取出来#print dictxfor dcx in dictx['userInfo']:#print dcxuhash=dcx['helloUrl'].encode('utf8').split('uhash=')[-1]crawled_time=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))#print uhashsql='''insert into shijijiayuan values(NULL,"%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")'''%(dcx['nickname'],dcx['age'],dcx['sex'],dcx['education'],dcx['height'],uhash,dcx['image'],dcx['marriage'],dcx['matchCondition'],\dcx['uid'],dcx['work_location'],dcx['work_sublocation'],dcx['shortnote'],crawled_time)#print '要插入的sql语句是：',sqlif not os.path.exists('images/%s_%s.jpg'%(dcx['nickname'],dcx['uid'])):passwhile(1):res2=''try:res2=requests.get(dcx['image'],headers=header,timeout=30)except Exception,e:print eif res2!='':breaktry:with open('images/%s_%s.jpg'%(dcx['nickname'],dcx['uid']),'wb') as f:f.write(res2.content)print '%s保存图片成功%s'%(dcx['uid'])except Exception,e:print '%s保存图片失败%s'%(dcx['uid'],str(e))sql2 = 'select 1 from shijijiayuan where uid=%s'%dcx['uid']mysqlx2 = MySql('localhost', 'root', '123456', 'test', 'utf8')mysqlx2.query(sql2)         is_exist = mysqlx2.cursor.fetchone()            ##判断下数据库有没有这个uid，uid是账号的唯一if is_exist==(1,):print '已存在%s'%dcx['uid']else:mysqlx2.query(sql)del mysqlx2pool = threadpool.ThreadPool(300)              ##用thredpool线程池开300线程
requestsx = threadpool.makeRequests(fun,[i for i in range(1,500)])
[pool.putRequest(req) for req in requestsx]
pool.wait()

执行这个后，就可以看到数据库里面会有很多会员信息了。

附上建表语句

CREATE TABLE `shijijiayuan` (
`id` int(10) NOT NULL AUTO_INCREMENT,
`nickname` varchar(20) DEFAULT NULL,
`age` int(5) DEFAULT NULL,
`sex` varchar(5) DEFAULT NULL,
`education` varchar(30) DEFAULT NULL,
`height` int(5) DEFAULT NULL,
`uhash` varchar(50) DEFAULT NULL,
`image` varchar(160) DEFAULT NULL,
`marriage` varchar(10) DEFAULT NULL,
`matchCondition` varchar(200) DEFAULT NULL,
`uid` int(15) DEFAULT NULL,
`work_location` varchar(10) DEFAULT NULL,
`work_sublocation` varchar(10) DEFAULT NULL,
`shortnote` varchar(1000) DEFAULT NULL,
`crawled_time` datetime DEFAULT NULL,
`last_hello_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`hello_times` int(3) DEFAULT '0',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8

这个是下载的图片。

第二部分是对会员发送信息。

#-*-coding:utf-8-*-
import requests,re,json,time,threadpool,os,random
from concurrent.futures import ThreadPoolExecutor
from mydba import MySql
from gevent import monkey
from greetings import greeting_listheader={'Cookie':'guider_quick_search=on; SESSION_HASH=yyyyyyya6f4b944ea522fb669bd758f17ea00; FROM_BD_WD=%25E4%25B8%2596%25E7%25BA%25AA%25E4%25BD%25B3%25E7%25BC%2598; FROM_ST_ID=416640; FROM_ST=.jiayuan.com; REG_ST_ID=15; REG_ST_URL=http://bzclk.baidu.com/adrc.php?t=06KL00c00f7t0wC0Gfum0QkHAsjb97wu00000PNeYH300000v9ZrmW.THL2sQ1PEPZRVfK85yF9pywd0Znqrj6vPWm1nH0snj0sm1N9PsKd5HTYwHTvrRmLfH9anDmLwHFjfRwArj6sfRcLPWTkn1610ADqI1YhUyPGujYzrH0dnWTYnHckFMKzUvwGujYkP6K-5y9YIZ0lQzqzuyT8ph-9XgN9UB4WUvYETLfE5v-b5HfkPWmYnaudThsqpZwYTjCEQLILIz4Jpy74Iy78QhPEUfKWThnqPWfdns&tpl=tpl_10762_15668_1&l=1053916117&attach=location%3D%26linkName%3D%25E6%25A0%2587%25E9%25A2%2598%26linkText%3D%25E4%25B8%2596%25E7%25BA%25AA%25E4%25BD%25B3%25E7%25BC%2598%25E7%25BD%2591%25EF%25BC%2588Jiayuan.com%25EF%25BC%2589%253A2017%25EF%25BC%258C%26xp%3Did(%2522m78828180%2522)%252FDIV%255B1%255D%252FDIV%255B1%255D%252FDIV%255B1%255D%252FH2%255B1%255D%252FA%255B1%255D%26linkType%3D%26checksum%3D191&ie=UTF-8&f=8&tn=baidu&wd=%E4%B8%96%E7%BA%AA%E4%BD%B3%E7%BC%98&oq=%E4%B8%96%E7%BA%AA%E4%BD%B3%E7%BC%98&rqlang=cn; user_access=1; _gscu_1380850711=98711257z5x28w21; _gscbrs_1380850711=1; save_jy_login_name=xxxxx; last_login_time=1498711280; user_attr=000000; pclog=%7B%22160961843%22%3A%221498711275466%7C1%7C0%22%7D; IM_S=%7B%22IM_CID%22%3A7816174%2C%22IM_SV%22%3A%22123.59.161.3%22%2C%22svc%22%3A%7B%22code%22%3A0%2C%22nps%22%3A0%2C%22unread_count%22%3A%221%22%2C%22ocu%22%3A0%2C%22ppc%22%3A0%2C%22jpc%22%3A0%2C%22regt%22%3A%221486465367%22%2C%22using%22%3A%2240%2C33%2C2%2C%22%2C%22user_type%22%3A%2210%22%2C%22uid%22%3A160961843%7D%2C%22m%22%3A12%2C%22f%22%3A0%2C%22omc%22%3A0%7D; REG_REF_URL=http://www.jiayuan.com/msg/?from=menu; PHPSESSID=8902adb4512b9e8904156c8b4267dc4c; stadate1=159961843; myloc=44%7C4403; myage=xx; PROFILE=160961843%3A%25E9%25A3%258E%3Am%3Aat1.jyimg.com%2F42%2Ffb%2F4558c12f370e446240ba148bcac4%3A1%3A%3A1%3A4558c12f3_2_avatar_p.jpg%3A1%3A1%3A61%3A10; mysex=m; myuid=15xxxx; myincome=40; mylevel=2; main_search:160xxxx=%7C%7C%7C00; RAW_HASH=suldoq0mMteEqQAs3qWKnMx-9sll5N8EDN-tu7bwGBA%2AtITMUwUbnL59YPR8ydVDXCxjt19iNPlu8kzi-jp8cU%2AzbhYS1iuOJyt%2APg32BhCebEE.; COMMON_HASH=424xxxx446240ba148bcac4fb; IM_CS=2; IM_ID=3; is_searchv2=1; pop_time=1498713117447; IM_CON=%7B%22IM_TM%22%3A1498713128148%2C%22IM_SN%22%3A9%7D; IM_M=%5B%5D; IM_TK=1498713324458','User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36',
}mysqlx = MySql('localhost', 'root', '123456', 'test', 'utf8')mysqlx.query("SELECT nickname,uhash FROM shijijiayuan order by id desc  limit 10,5")   #SELECT * FROM shijijiayuan where hello_times=0 limit 1,1 #select语句选取几个账号发信
fetchallx= mysqlx.cursor.fetchall()
nickname_uhash_list=[ft for ft in fetchallx] url2='http://www.jiayuan.com/msg/dosend.php?type=hello&randomfrom=4'
url3='http://www.jiayuan.com/165508547'
print nickname_uhash_list
def hello(nickname,uhash):datax2={'textfield':'嗨。%s妹子。'%(nickname)+random.choice(greeting_list),   ##问候语是一个列表，随机发送一个信息给他。'list':'','category':'2','xitong_zidingyi_wenhouyu':'1','xitong_zidingyi_wenhouyu':'0','new_type':'1','pre_url':'','sendtype':'20','hellotype':'hello','pro_id':'0','new_profile':'3','to_hash':uhash,       #9c248d2c96868695640e76ed74e9d3b0   f11e574d44c7967b88afb9940392c14b uhash的值决定对谁发送信息，不是uid来决定，接口中没有uid参数'fxly':'search_v2','tj_wz':'none','need_fxtyp_tanchu':'0','self_pay':'0','fxbc':'0','cai_xin':'0','zhuanti':'0','liwu_nofree':'0','liwu_nofree_id':'88',}try:contentx='空'resx= requests.post(url2,data=datax2,headers=header)contentx= resx.contentprint contentxif  '发送成功' in contentx:    ##如果发送信息成功，就把数据库中的最后发信时间更新下，然后把发信次数加1，以便于之后用slect语句选定没有发信的人对其发信mysqlx2 = MySql('localhost', 'root', '123456', 'test', 'utf8')#update student set score=score+1 where id = 1sql='''UPDATE shijijiayuan  SET hello_times=hello_times+1 WHERE  uhash="%s"'''%(uhash)#print sql
            mysqlx2.query(sql)del  mysqlx2except Exception,e:print epoolx = ThreadPoolExecutor(max_workers=10)  ##这也是多线程发信，世纪佳缘网站发信接口，响应时间非常巨大，浪费时间，用多线程Threadpoolexurot非常好的包，是未来要做的事，把一切东西扔到这里面，本篇没有体现出比threadpool好的优点。
[poolx.submit(hello,i[0],i[1]) for i in nickname_uhash_list]

执行完这个脚本后，在网页或者app中就可以看到发的信息了。

这个是打了招呼之后表的变化。

没有什么难度，就是要分析，首先是分析账号信息在哪里，是ajax，返回的是json加了一些字符，把json提取出来，转成字典，比较好弄。

然后就是要分析发信接口是哪一个url，这个根据规律猜测下然后查看下post的值就能找到，然后就是找出接口中的关键字段，关键字段不多，uhash是决定发送给谁（尝试几下就可以），textfield是发送的内容。

本文的登录是headers中的cookie字段携带。

世纪佳缘信息爬取存储到mysql,下载图片到本地，从数据库选取账号对其发送消息更新发信状态相关推荐

mysql存储爬虫图片_世纪佳缘信息爬取存储到mysql,下载图片到本地，从数据库选取账号对其发送消息更新发信状态...
利用这种方法,可以把所有会员信息存储下来,多线程发信息,10秒钟就可以对几百个会员完成发信了. 首先是筛选信息后爬取账号信息, #-*-coding:utf-8-*- importrequests,r ...
世纪佳缘信息爬取存储到mysql,下载图片到本地，从数据库选取账号对其发送消息更新发信状态...
利用这种方法,可以把所有会员信息存储下来,多线程发信息,10秒钟就可以对几百个会员完成发信了. 首先是筛选信息后爬取账号信息, #-*-coding:utf-8-*- import requests, ...
python3爬虫爬取百度贴吧下载图片
python3爬虫爬取百度贴吧下载图片学习爬虫时没事做的小练习. 百度对爬虫还是很友好的,在爬取内容方面还是较为容易. 可以方便各位读者去百度贴吧一键下载每个楼主的图片,至于是什么类型的图片,就看你 ...
用selenium 爬取世纪佳缘信息
仅供参考,以学习为主一.用selenium获取用户的个人连接地址,并保存到excel中脚本名:url.py import requests from bs4 import BeautifulSou ...
36-Scrapy框架-西刺网代理爬取+存储到MySQL
目的: 爬取西刺网https://www.xicidaili.com/nn下3799页的代理相关信息将爬取的信息存储到mysql数据库采用基于spider类的Scrapy框架爬虫,所以在构建爬取页 ...
分析Ajax爬取今日头条，下载图片
这几天打算趁着寒假时间把之前学的爬虫捡回来,发现现在今日头条改了一丢丢.与崔庆才老师有关分析Ajax爬取今日头条街拍美图已经有点出入. 获取搜索结果的ur的方法与崔庆才老师是一样的,这里就不细说了,不 ...
python 网络爬虫爬取落网期刊内容下载图片音乐
1.实现方案 1.1 采用多进程创建多个爬虫对象爬虫对象主要由获取网页内容,分析网页内容,下载图片,下载音乐这四个线程组成 1.2 进程及线程数根据网络情况设置 1.3 图片保存 ...
爬取贴吧上的图片到本地
在浏览网页时想保存一些图片,通常方法: *1. 鼠标右键,另存为: *2.无另存为选项,截图保存: *3.右键查看源代码以python为例,爬取贴吧上大量图片,用方法3实现如下: 获取整个页面数据 ...
手把手教你用Scrapy 爬取斗鱼妹子头像下载图片
项目展示: 项目结构: 成果: 1.新建项目确保安装了所用到的库 scrapy image Pillow pip install scrapy pip install image pip insta ...

世纪佳缘信息爬取存储到mysql,下载图片到本地，从数据库选取账号对其发送消息更新发信状态

世纪佳缘信息爬取存储到mysql,下载图片到本地，从数据库选取账号对其发送消息更新发信状态相关推荐

最新文章

热门文章