"""作者：李舵日期：2019-4-27功能：抓取豆瓣电影top250版本：V1.0
"""import time
import json
import requests
from bs4 import BeautifulSoup
from requests.exceptions import RequestExceptiondef get_one_page(url):try:headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36'}response = requests.get(url, headers=headers)if response.status_code == 200:return response.textreturn Noneexcept RequestException:return Nonedef parse_one_page(html):soup = BeautifulSoup(html, 'lxml')ol_list = soup.find('ol', {'class': 'grid_view'})li_list = ol_list.find_all('li')for i in range(25):move_value = li_list[i]yield {'index': move_value.find('em', {'class': ''}).text.strip(),'title': move_value.find('span', {'class': 'title'}).text.strip(),'actor': move_value.find('p', {'class': ''}).text.strip(),'score': move_value.find('span', {'class': 'rating_num'}).text.strip()}def write_to_file(content):with open('result.txt', 'a', encoding='utf-8') as f:print(type(json.dumps(content)))f.write(json.dumps(content, ensure_ascii=False)+'\n')def main(start):url = 'https://movie.douban.com/top250?start=' + str(start)html = get_one_page(url)for item in parse_one_page(html):print(item)write_to_file(item)if __name__ == '__main__':for i in range(0,250,25):main(start=i)time.sleep(1)

"""
作者：李舵
日期：2019 - 4 - 8
功能：抓取豆瓣电影top250
版本：V2.0
"""import re
import time
import requests
from requests.exceptions import RequestExceptiondef get_one_page(url):try:headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36'}response = requests.get(url, headers=headers)if response.status_code == 200:return response.textreturn Noneexcept RequestException:return Nonedef parse_one_page(html):pattern = re.compile(u'<div.*?class="item">.*?'+ u'<div.*?class="pic">.*?'+ u'<em.*?class="">(.*?)</em>.*?'+ u'<div.*?class="info">.*?'+ u'<span.*?class="title">(.*?)</span>.*?'+ u'<span.*?class="other">(.*?)</span>.*?'+ u'<div.*?class="bd">.*?'+ u'<p.*?class="">.*?'+ u'导演:\s(.*?)\s.*?<br>'+ u'(.*?) / '+ u'(.*?) / (.*?)</p>.*?'+ u'<div.*?class="star">.*?'+ u'<span.*?class="rating_num".*?property="v:average">'+ u'(.*?)</span>.*?'+ u'<span>(.*?)人评价</span>.*?'+ u'<span.*?class="inq">(.*?)</span>', re.S)movies = re.findall(pattern, html)movie_list = []for movie in movies:movie_list.append([movie[0],movie[1],movie[2].lstrip(' / '),movie[3],movie[4].lstrip(),movie[5],movie[6].strip(),movie[7],movie[8],movie[9]])return movie_listdef write_to_file(movie_list):with open('top_250.txt', 'w', encoding='utf-8',) as f:for movie in movie_list:f.write('电影排名：' + movie[0] + '\n')f.write('电影名称：' + movie[1] + '\n')f.write('电影别名：' + movie[2] + '\n')f.write('导演：' + movie[3] + '\n')f.write('上映年份：' + movie[4] + '\n')f.write('制作国家/地区：' + movie[5] + '\n')f.write('电影类别：' + movie[6] + '\n')f.write('评分：' + movie[7] + '\n')f.write('参评人数：' + movie[8] + '\n')f.write('简短影评：' + movie[9] + '\n')f.write('\n')print('成功写入文件，共有%d条记录……' % len(movie_list))f.close()def main(start):url = 'https://movie.douban.com/top250?start=' + str(start)html = get_one_page(url)movie_list = parse_one_page(html)write_to_file(movie_list)if __name__ == '__main__':for i in range(0, 250, 25):main(start=i)time.sleep(1)