利用Python爬取电影网站

#!/usr/bin/env python
#coding = utf-8
'''
本爬虫是用来爬取6V电影网站上的电影资源的一个小脚本程序，爬取到的电影链接会通过网页的形式显示出来
'''
import requests
import re
from bs4 import BeautifulSoup as bs
from queue import Queue
from other import getUser_Agent
import threading
import sys
import timeheaders = getUser_Agent.getUser_Agent()class Movielinks(threading.Thread):def __init__(self,que,filepath,totalcount,starttime):threading.Thread.__init__(self)self._que = queself._filepath = filepathself._totalcount = totalcountself._starttime = starttimedef run(self):try:while not self._que.empty():url = self._que.get()threading.Thread(target=self.showdetail).start()self.spider(url)except:print('error--->def run(self):')def spider(self,url):try:r = requests.get(url,headers)file = open(self._filepath,'a+')if r.status_code == 200:soup = bs(r.content.decode('gbk'),'html.parser')links = soup.find_all('td')title = re.search(r'《(.*?)》',soup.title.string)# print(title.group())for link in links:if '下载帮助' not in str(link):hrefs = link.find_all('a')# print(link)# print('**' * 100)# mima = re.search(r'</a>(.*?)</td>',str(link))# print(mima.group())for href in hrefs:if 'ed2k' in href['href'] or '.torrent' in href['href'] or 'thunder' in href['href'] or 'ftp' in href['href']:# sys.stdout.write('\r'+'\t\t迅雷下载链接：%s'%href['href'])file.write('<li style="list-style-type: decimal;">迅雷下载链接：<a href="%s" target="%s">%s</a></li>' % (href['href'],'_blank',title.group()))file.write('\n<br>')elif 'baidu' in href['href'] and re.search(r'密码：\w{4}',str(link)):wangpan_password = re.search(r'密码：\w{4}',str(link))# sys.stdout.write('\r'+'\t\t百度网盘下载链接：%s    网盘%s'%(href['href'],wangpan_password.group()))file.write('<li style="list-style-type: decimal;">百度网盘链接：<a href="%s" target="%s">%s</a><b>%s</b></li>' % (href['href'], '_blank', title.group(),wangpan_password.group()))file.write('\n<br>')elif '正版观看' in str(href):# print('\t\t %s  该影片需要正版观看！！' % title.group())file.write('<b><font color="#00ff00">%s需要正版版权才能观看</font></b>' % title.group())file.write('\n<br>')else:file.write('<b><font color="#ff0000">%s 的该条链接无法正常爬取，尽情谅解</font></b>' % title.group())file.write('\n<br>')# sys.stdout.write('\r'+'\t\terror--->def spider(self,url)-in:%s'%title.group())# print()file.write('<br>')file.close()else:print('%s 该磁力链接已坏！！' % url)except:# print('error--->def spider(self,url)-out:%s' % title.group())file.write('<b><font color="#ff0000">%s 无法正常爬取，尽情谅解</font></b>' % title.group())def showdetail(self):usetime = time.time() - self._starttimeper = 100 - (float(self._que.qsize())/float(self._totalcount)) * 100sys.stdout.write('\r'+'下载链接进度：%.2f %s  用时：%.3f 秒' % (float(per),'%',float(usetime)))def getMovieCount(searchid):url = 'http://www.6vhao.tv/e/search/result/?searchid=' + str(searchid)r = requests.get(url,headers)soup = bs(r.content,'html.parser')divs = soup.find(name='div',attrs='channellist')pages = re.search(r'\d+',divs.h2.string)print('%s 一共有%2.f页'%(divs.h2.string,float(pages.group())/20))def getWantPagesUrls(url,startpage,endpage,searchid):urls = []for i in range(startpage,endpage+1):link = 'http://www.6vhao.tv/e/search/result/index.php?page='+str(i)+'&searchid='+str(searchid)urls.append(link)return urlsdef getpagesLinks(urls): # 返回一个电影名字和链接对应的字典pageslink_dic = {}moviename_list = []moviehref_list = []for url in urls:r = requests.get(url,headers)soup = bs(r.content,'html.parser')divs = soup.find_all('div',class_='listimg')for div in divs:moviehref = div.find('a')['href']moviename = div.find('a').img['alt']moviehref_list.append(moviehref)# moviename_list.append(moviename)# for i,name in zip(range(len(moviename_list)),moviename_list):#     pageslink_dic[name] = moviehref_list[i]# print(moviehref_list)return moviehref_listdef getqueue(urls):que = Queue()for url in urls:que.put(url)return quedef main():tishi = '''*6v电影网站电影链接爬虫*#要搜索的常用关键字代码#1.韩国--->1857732.日本--->1856913.国产--->1865044.美国--->1871815.英国--->1881616.香港--->1884617.喜剧--->1854418.恐怖--->1871939.悬疑--->19022610.记录片--->18796311.科幻--->18986612.战争--->18783013.动画--->187978'''print(tishi)keywords = {1:185773,2:185691,3:186504,4:187181,5:188161,6:188461,7:185441,8:187193,9:190226,10:187963,11:189866,12:187830,13:187978}threads = []url = 'http://www.6vhao.tv/e/search/result/index.php?page='searchid = input('请输入搜索关键字序号>>>')print('正在计算电影总量... ...')getMovieCount(keywords[int(searchid)])startpage = int(input('请输入起始页码>>>'))endpage = int(input('请输入终止页码>>>'))filepath = input('请输入电影链接要保存的文件路径（扩展名是：.html）')print('The program is running,Please waiting... ...')urls = getpagesLinks(getWantPagesUrls(url,startpage,endpage,keywords[int(searchid)]))que = getqueue(urls)print('本次下载行为：从%d页到%d页，有%d部电影正在下载...'%(startpage,endpage,que.qsize()))thread_count = que.qsize()starttime = time.time()for i in range(thread_count):threads.append(Movielinks(que,filepath,thread_count,starttime))for t in threads:t.start()for t in threads:t.join()if __name__ == '__main__':# ch = int(input('请输入选项（负数退出）>>>'))# while ch >0:#     main()#     ch = int(input('请输入选项（负数退出）>>>'))main()

在爬取网页时，需要模拟浏览器的操作，所以在爬取数据时需要模拟一个user-agent ,相关代码如下：

from random import randintdef getUser_Agent():headers = ['Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',   #safari 5.1 – MAC'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',  #safari 5.1 – Windows'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',   #IE 9.0'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',   #IE 8.0'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',   #IE 7.0'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',   # IE6.0'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',   #Firefox 4.0.1 – MAC'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',     #Firefox 4.0.1 – Windows'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',     #Opera 11.11 – MAC'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',  #Opera 11.11 – Windows'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',    #Chrome 17.0 – MAC'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',  #傲游（Maxthon）'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)',  #腾讯TT'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',  #世界之窗（The World） 2.x'ozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)',  #世界之窗（The World） 3.x'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)',  #搜狗浏览器 1.x'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)',  #360浏览器'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)',     #Avant'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',     #Green Browser##移动端用户代理'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5',     #safari iOS 4.33 – iPhone'Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5',     #safari iOS 4.33 – iPod Touch'Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5',     #safari iOS 4.33 – iPad'Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1',  #Android N1'MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1',  #Android QQ浏览器 For android'Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10',  #Android Opera Mobile'Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+',  #Android Pad Moto Xoom'Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0',  #WebOS HP Touchpad'Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124',  #Nokia N97'Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)',  #Windows Phone Mango'UCWEB7.0.2.37/28/999',  #UC无'Openwave/ UCWEB7.0.2.37/28/999',  #UCOpenwave'Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999',  #UC Opera]return {'User-Agent':headers[randint(0,len(headers)-1)]}

posted @ 2018-10-14 11:04 东京的樱花在巴黎飘下阅读(...) 评论(...) 编辑收藏

【PYTHON】利用Python爬取电影网站相关推荐

python利用bs4爬取外国高清图片网站
python利用bs4爬取外国高清图片网站爬取高清图片爬取高清图片 import re import requests from bs4 import BeautifulSoup import o ...
python爬电影_使用Python多线程爬虫爬取电影天堂资源
最近花些时间学习了一下Python,并写了一个多线程的爬虫程序来获取电影天堂上资源的迅雷下载地址,代码已经上传到GitHub上了,需要的同学可以自行下载.刚开始学习python希望可以获得宝贵的意见. ...
Python爬虫实战爬取租房网站2w+数据-链家上海区域信息（超详细）
Python爬虫实战爬取租房网站-链家上海区域信息(过程超详细) 内容可能有点啰嗦大佬们请见谅后面会贴代码带火们有需求的话就用吧正好这几天做的实验报告就直接拿过来了,我想后面应该会有人用的到吧 ...
Python利用Scrapy爬取前程无忧
** Python利用Scrapy爬取前程无忧 ** 一.爬虫准备 Python:3.x Scrapy PyCharm 二.爬取目标爬取前程无忧的职位信息,此案例以Python为关键词爬取相应的职位 ...
python爬虫实战-爬取视频网站下载视频至本地(selenium)
#python爬虫实战-爬取视频网站下载视频至本地(selenium) import requests from lxml import etree import json from selenium ...
python翻页爬取电影天堂网电影的磁力链接
python翻页爬取电影天堂网电影的磁力链接案例目的: 介绍如何通过首页的url提取详情页的url以及如何通过xpath语法提取详情页的数据. 代码功能: 输入要爬取的页数,自动保存电影的名称以及对 ...
增量爬取电影网站2级详情页面电影名称和下载链接
增量爬取电影网站2级详情页面电影名称和下载链接需求步骤需求一级页面抓取:电影详情页链接二级页面抓取:电影名称和电影下载链接增量获取:网站中电影资源后将更新的数据爬取下来步骤确定响应内容 ...
python爬取电影网站存储于数据库_Python零基础爬虫教程（实战案例爬取电影网站资源链接）...
前言好像没法添加链接,文中的链接只能复制到浏览器查看了这篇是我写在csdn的,那里代码格式支持更好,文章链接 https://blog.csdn.net/d497465762/article/de ...
python爬网站的题库_Python零基础爬虫教程（实战案例爬取电影网站资源链接）
前言好像没法添加链接,文中的链接只能复制到浏览器查看了这篇是我写在csdn的,那里代码格式支持更好,文章链接 https://blog.csdn.net/d497465762/article/de ...
python爬取web漫画网站_[Python爬虫学习]利用selenuim爬取漫画网站
注意事项版本 Python版本为 Python-3.8.3 系统为 Windows10 浏览器为 Firefox-77.0 前置 \(selenium\) \(BeautifulSoup\) \(r ...

【PYTHON】利用Python爬取电影网站

利用Python爬取电影网站

【PYTHON】利用Python爬取电影网站相关推荐

最新文章

热门文章