思路分析

1、获取每一个页面的URL地址

首页的URL：https://www.dytt8.net/html/gndy/dyzz/index.html
以后每一个页面的URL：https://www.dytt8.net/html/gndy/dyzz/list_23_%d.html（其中的%d应该替换为相应的页码）

url = 'https://www.dytt8.net/html/gndy/dyzz/list_23_%d.html' % i

2、获取电影详情页面的URL

利用XPath提取相应的URL即可

def get_url(page_url):try:response = requests.get(page_url, headers=headers)page_url_list = []except :print(page_url+"请求失败!")return []html = etree.HTML(response.text)a = html.xpath("//a[@class='ulink']")for it in a:href_list = it.xpath("@href")if len(href_list) > 0:href = href_list[0]page_url_list.append(HOST+href)print(page_url_list)return page_url_list

3、爬取电影详情页面

根据拿到的电影详情页面的URL发送请求

4、从页面中提取信息

利用XPath提取需要的信息


def get_detail(page_url):try:r = requests.get(page_url, headers=headers)except :print(page_url+"请求失败")return {}r.encoding = 'gbk'html = etree.HTML(r.text)p_list = html.xpath("//p")if len(p_list) == 0:return {}else:p = p_list[0]img_list = p.xpath("./img/@src")if len(img_list) == 0:img = 'null'else:img = img_list[0]download_url_list = p.xpath("./a/@href")if len(download_url_list) == 0:download_url = "null"else:download_url = download_url_list[0]movie_info = {'image': img}info = p.xpath("./text()")if len(info) == 0:return {}for index, item in enumerate(info):if item.startswith('◎译　　名'):translated_names = parse_info('◎译　　名', item)movie_info['translated_names'] = translated_nameselif item.startswith('◎片　　名'):title = parse_info('◎片　　名', item)movie_info['title'] = titleelif item.startswith('◎年　　代'):year = parse_info('◎年　　代', item)movie_info['year'] = yearelif item.startswith('◎产　　地'):place = parse_info('◎产　　地', item)movie_info['place'] = placeelif item.startswith('◎类　　别'):category = parse_info('◎类　　别', item)movie_info['category'] = categoryelif item.startswith('◎语　　言'):language = parse_info('◎语　　言', item)movie_info['language'] = languageelif item.startswith('◎上映日期'):release_date = parse_info('◎上映日期', item)movie_info['release_date'] = release_dateelif item.startswith('◎IMDb评分'):IMDB_score = parse_info('◎IMDb评分', item)movie_info['IMDB_score'] = IMDB_scoreelif item.startswith('◎豆瓣评分'):score = parse_info('◎豆瓣评分', item)movie_info['score'] = scoreelif item.startswith('◎片　　长'):duration = parse_info('◎片　　长', item)movie_info['duration'] = durationelif item.startswith('◎导　　演　'):director = parse_info('◎导　　演　', item)movie_info['director'] = directorelif item.startswith('◎编　　剧'):screenwriter = parse_info('◎编　　剧', item)movie_info['screenwriter'] = screenwriterelif item.startswith('◎主　　演'):actor = [parse_info('◎主　　演', item)]for i in range(index + 1, len(info)):if info[i].startswith('◎'):breakactor.append(info[i].strip())movie_info['actor'] = actorelif item.startswith('◎标　　签'):label = parse_info('◎标　　签', item)movie_info['label'] = labelelif item.startswith('◎简　　介 '):information = ""for i in range(index + 1, len(info)):if info[i].startswith('◎'):breakinformation += info[i].strip()movie_info['information'] = informationmovie_info['download_url'] = download_urlif len(movie_info) < 3:return {}print(movie_info)return movie_info

5、将提取到的信息保存到文件

将列表保存成CSV文件即可

def save_to_file(movie_list, mode, header):df = pd.DataFrame(movie_list)df.to_csv('./result/Movie_Tian.csv', index=False, encoding='utf_8_sig', mode=mode, header=header)

完整代码

# ！/usr/bin/env python
# —*— coding: utf-8 —*—
# @Time:    2020/2/1 21:36
# @Author:  Martin
# @File:    Movie_Tian.py
# @Software:PyCharm
import requests
from lxml import etree
import pandas as pd
HOST = 'https://www.dytt8.net'
index_url = 'https://www.dytt8.net/html/gndy/dyzz/index.html'
raw_url = 'https://www.dytt8.net/html/gndy/dyzz/list_23_%d.html'
headers = {'Referer': 'https://www.dytt8.net/','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36'
}def get_url(page_url):try:response = requests.get(page_url, headers=headers)page_url_list = []except :print(page_url+"请求失败!")return []html = etree.HTML(response.text)a = html.xpath("//a[@class='ulink']")for it in a:href_list = it.xpath("@href")if len(href_list) > 0:href = href_list[0]page_url_list.append(HOST+href)print(page_url_list)return page_url_listdef get_detail(page_url):try:r = requests.get(page_url, headers=headers)except :print(page_url+"请求失败")return {}r.encoding = 'gbk'html = etree.HTML(r.text)p_list = html.xpath("//p")if len(p_list) == 0:return {}else:p = p_list[0]img_list = p.xpath("./img/@src")if len(img_list) == 0:img = 'null'else:img = img_list[0]download_url_list = p.xpath("./a/@href")if len(download_url_list) == 0:download_url = "null"else:download_url = download_url_list[0]movie_info = {'image': img}info = p.xpath("./text()")if len(info) == 0:return {}for index, item in enumerate(info):if item.startswith('◎译　　名'):translated_names = parse_info('◎译　　名', item)movie_info['translated_names'] = translated_nameselif item.startswith('◎片　　名'):title = parse_info('◎片　　名', item)movie_info['title'] = titleelif item.startswith('◎年　　代'):year = parse_info('◎年　　代', item)movie_info['year'] = yearelif item.startswith('◎产　　地'):place = parse_info('◎产　　地', item)movie_info['place'] = placeelif item.startswith('◎类　　别'):category = parse_info('◎类　　别', item)movie_info['category'] = categoryelif item.startswith('◎语　　言'):language = parse_info('◎语　　言', item)movie_info['language'] = languageelif item.startswith('◎上映日期'):release_date = parse_info('◎上映日期', item)movie_info['release_date'] = release_dateelif item.startswith('◎IMDb评分'):IMDB_score = parse_info('◎IMDb评分', item)movie_info['IMDB_score'] = IMDB_scoreelif item.startswith('◎豆瓣评分'):score = parse_info('◎豆瓣评分', item)movie_info['score'] = scoreelif item.startswith('◎片　　长'):duration = parse_info('◎片　　长', item)movie_info['duration'] = durationelif item.startswith('◎导　　演　'):director = parse_info('◎导　　演　', item)movie_info['director'] = directorelif item.startswith('◎编　　剧'):screenwriter = parse_info('◎编　　剧', item)movie_info['screenwriter'] = screenwriterelif item.startswith('◎主　　演'):actor = [parse_info('◎主　　演', item)]for i in range(index + 1, len(info)):if info[i].startswith('◎'):breakactor.append(info[i].strip())movie_info['actor'] = actorelif item.startswith('◎标　　签'):label = parse_info('◎标　　签', item)movie_info['label'] = labelelif item.startswith('◎简　　介 '):information = ""for i in range(index + 1, len(info)):if info[i].startswith('◎'):breakinformation += info[i].strip()movie_info['information'] = informationmovie_info['download_url'] = download_urlif len(movie_info) < 3:return {}print(movie_info)return movie_infodef parse_info(string, item):return item.replace(string, "").strip()def spider():# 指定结束页码（目前共208页），这里爬取前10页end_index = 10all_url = get_url(index_url)for index, page_url in enumerate(all_url):if index == 0:movie_list = [get_detail(page_url)]save_to_file(movie_list, 'w', True)else:movie_list = [get_detail(page_url)]save_to_file(movie_list, 'a', False)for i in range(2, end_index+1):all_url = get_url(raw_url % i)for page_url in all_url:movie_list = [get_detail(page_url)]save_to_file(movie_list, 'a', False)def save_to_file(movie_list, mode, header):df = pd.DataFrame(movie_list)df.to_csv('./result/Movie_Tian.csv', index=False, encoding='utf_8_sig', mode=mode, header=header)if __name__ == '__main__':spider()

结果展示

注意事项

1、利用requests库发送请求时，为避免网络状况不佳导致程序崩溃，可以利用tr语句y来增强程序的健壮性。
2、利用XPath提取数据时，由于不能保证每个页面都完全一样，因此，应考虑提取不到数据的情况，避免出现列表索引越界的现象。

虐猫人薛定谔i 2020年2月2日写于家中

爬虫实战——爬取电影天堂的电影详情页信息相关推荐

Python爬虫实战爬取租房网站2w+数据-链家上海区域信息（超详细）
Python爬虫实战爬取租房网站-链家上海区域信息(过程超详细) 内容可能有点啰嗦大佬们请见谅后面会贴代码带火们有需求的话就用吧正好这几天做的实验报告就直接拿过来了,我想后面应该会有人用的到吧 ...
第一次写爬虫程序爬取豆瓣5W条电影数据
第一次写爬虫程序爬取豆瓣5W条电影数据最近工作比较不是很忙,想到之前使用httpclient和jsoup爬取过一次豆瓣电影TOP250,但总觉得数据量太小,不过瘾.于是趁着最近不是很忙的机会,重新写 ...
node 爬虫实战 - 爬取拉勾网职位数据
node 爬虫实战 - 爬取拉勾网职位数据,主要想把数据用于大数据学习,到时候大数据分析可以自己分析一下职位的情况,和比较一些我现在的职位在深圳乃至全国的开发人员水平. 涉及到的技术栈:node.j ...
python爬虫实战---爬取大众点评评论
python爬虫实战-爬取大众点评评论(加密字体) 1.首先打开一个店铺找到评论很多人学习python,不知道从何学起. 很多人学习python,掌握了基本语法过后,不知道在哪里寻找案例上手. 很多 ...
python爬虫实战-爬取视频网站下载视频至本地(selenium)
#python爬虫实战-爬取视频网站下载视频至本地(selenium) import requests from lxml import etree import json from selenium ...
python爬虫实战-爬取微信公众号所有历史文章 - (00) 概述
http://efonfighting.imwork.net 欢迎关注微信公众号"一番码客"获取免费下载服务与源码,并及时接收最新文章推送. 最近几年随着人工智能和大数据的兴起,p ...
Scrapy 爬虫实战-爬取字幕库
Scrapy 爬虫实战-爬取字幕库 1.首先,创建Scrapy框架创建工程 scrapy startproject zimuku创建爬虫程序 cd zimuku scrapy genspider z ...
python爬虫：爬取猎聘网站职位详情
python爬虫:爬取猎聘网站职位详情第一次学习python,也是刚开始学习爬虫,完成的第一个实例,记录一下. baseurl.py # @author centao # @time 2020.10 ...
Python3简单爬虫：爬取猫眼评分top100电影
Python3:用xpath库爬取猫眼评分top100电影在看<Python3 网络爬虫开发实战中>一书学习时,书中第三章例子用re正则匹配来爬取电影的所需数据,虽然爬取速度快,效率好, ...
python爬虫实战--爬取猫眼专业版-实时票房
小白级别的爬虫入门最近闲来无事,发现了猫眼专业版-实时票房,可以看到在猫眼上映电影的票房数据,便验证自己之前学的python爬虫,爬取数据,做成.svg文件. 爬虫开始之前我们先来看看猫眼专业版- ...

爬虫实战——爬取电影天堂的电影详情页信息

爬取电影天堂的电影详情页信息