from bs4 import BeautifulSoup
import re, csv, urllib.request, urllib.parse, time, json, pickleclass url_manager(object):def __init__(self):self.new_urls = []  # 书籍上采用set()函数，主要是考虑到次函数的去重功能，但集合是无序的，导致不方便查找new_urls内的数据，且后序add(url)时已进行判定，不必要再使用set()self.old_urls = []def add_new_url(self, url):if url not in self.new_urls and url not in self.old_urls:self.new_urls.append(url)def add_new_urls(self, urls):if urls == None:returnfor url in urls:self.add_new_url(url)def have_new_url(self):return len(self.new_urls) != 0def get_new_url(self):data = self.new_urls.pop(0)  # 从第一个数据进行删除，逐一爬虫
self.old_urls.append(data)return dataclass url_download(object):def download(self, url):response = urllib.request.urlopen(url)data = response.read().decode()if data == None:print("no web")return Falsereturn dataclass url_scrapy(object):def get_data(self, source_url, source_data):#书中将票房的电影由于其json中关键字的差异，进行了分类，本程序中为了简单起见，只是读取一下关键字。
pattern = re.compile(r'=(.*?);')jsondata = pattern.findall(str(source_data))[0]jsondata2=json.loads(jsondata)movieresult={'MovienIDame': jsondata2['value']['movieRating']['MovieId'],'RatingFinal': jsondata2['value']['movieRating']['RatingFinal'],'movieTitle': jsondata2['value']['movieTitle'] }print(movieresult)return (source_url,movieresult)  # 没有搜索到url时，返回Nonedef get_sameurldata(self, source_url, source_data):a=[]pattern = re.compile(r'=(.*?);')jsonurl = pattern.findall(str(source_data))[0]jsonurl1=json.loads(jsonurl)movielist=jsonurl1['value']['movieList']for i in movielist:a.append(i['url'])return (a)  # 没有搜索到url时，返回Noneclass output_url(object):def output_scroe(self, root_url):  # 组建动态网址time0 = time.strftime("%Y%m%d%H%M%S11111", time.localtime())a = re.compile(r'/(\d+)/')urlnum = a.findall(root_url)url2 = "http%3A%2F%2Fmovie.mtime.com%2F" + urlnum[0] + '%2F'scroe_url = 'http://service.library.mtime.com/Movie.api?' \'Ajax_CallBack=true' \'&Ajax_CallBackType=Mtime.Library.Services' \'&Ajax_CallBackMethod=GetMovieOverviewRating' \'&Ajax_CrossDomain=1' \'&Ajax_RequestUrl=%s' \'&t=%s' \'&Ajax_CallBackArgument0=%s' % (url2, time0, urlnum[0])return scroe_urldef output_sameurl(self, root_url):  # 组建动态网址
time0 = time.strftime("%Y%m%d%H%M%S11111", time.localtime())a = re.compile(r'/(\d+)/')urlnum = a.findall(root_url)url2 = "http%3A%2F%2Fmovie.mtime.com%2F" + urlnum[0] + '%2F'same_url = 'http://service.library.mtime.com/Movie.api?' \'Ajax_CallBack=true' \'&Ajax_CallBackType=Mtime.Library.Services' \'&Ajax_CallBackMethod=GetSimilarRecommenMovieInfoByMovieId' \'&Ajax_CrossDomain=1' \'&Ajax_RequestUrl=%s' \'&t=%s' \'&Ajax_CallBackArgument0=%s' % (url2, time0, urlnum[0])return same_urlclass output_data(object):def data_save(self, data):with open('pachong.csv', "a+", encoding='utf-8') as f:f1 = csv.writer(f, lineterminator='\n')f1.writerows(data)class controllers(object):def __init__(self):self.manager = url_manager()self.download = url_download()self.scrapy = url_scrapy()self.output = output_data()self.scoreurl = output_url()def control(self, url):self.manager.add_new_url(url)num = 1data1 = 0while (1):if num > 200:breakelif self.manager.have_new_url():url_down = self.manager.get_new_url()score_url = self.scoreurl.output_scroe(url_down)same_url = self.scoreurl.output_sameurl(url_down)info = self.download.download(score_url)url_info=self.download.download(same_url)data1, url1 = self.scrapy.get_data(url_down, info)same_url1= self.scrapy.get_sameurldata(url_down, url_info)if data1 != None:self.output.data_save(data1)print(num, "is finished:", url_down)num += 1self.manager.add_new_urls(same_url1)else:print('has no url')breakif __name__ == "__main__":url = r'http://movie.mtime.com/225824/'a = controllers()a.control(url)

方案二：采用selenium技术进行加载动态网站信息，直接获取数据