.*?title="(.*?)".*?

(.*?)

.*?

(.*?)

'pattern=re.compile(re_, re.S)#film_list:[('霸王别姬','张国荣','1993')]

film_list =pattern.findall(html)

self.write_html(film_list)#存入csv文件-writerrows

defwrite_html(self, film_list):

L=[]

with open('maoyanfilm.csv', 'a',newline='') as f:#初始化写入对象,注意参数f不能忘

writer =csv.writer(f)for film infilm_list:

t=(

film[0].strip(),

film[1].strip(),

film[2].strip()[5:15]

)

self.num+= 1L.append(t)#writerow()参数为列表

writer.writerows(L)print(L)#def write_html(self,film_list):

#with open('maoyanfilm.csv','a') as f:

##初始化写入对象,注意参数f不能忘

#writer=csv.writer(f)

#for film in film_list:

#L=[

#film[0].strip(),

#film[1].strip(),

#film[2].strip()[5:15]

#]

#self.num+=1

## writerow()参数为列表

#writer.writerow(L)

defmain(self):for offset in range(0, 91, 10):

url=self.url.format(offset)

self.get_html(url)

time.sleep(random.randint(1, 2))print('共抓取数据', self.num, "部")if __name__ == '__main__':

start=time.time()

spider=MaoyanSpider()

spider.main()

end=time.time()print('执行时间:%.2f' % (end - start))

writeow方法

ua_list =['Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)','Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0','Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1 QQBrowser/6.9.11079.201','Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.9.168 Version/11.50',

]

useragents.py配置文件

二数据持久化存储 - MySQL数据库

1.在数据库中建库建表

#连接到mysql数据库

mysql -h127.0.0.1 -uroot -p123456#建库建表

create database maoyandb charset utf8;

use maoyandb;

create table filmtab(

name varchar(100),

star varchar(300),

time varchar(50)

)charset=utf8;

2.回顾pymysql基本使用

importpymysql#创建2个对象

db = pymysql.connect('localhost','root','123456','maoyandb',charset='utf8')

cursor=db.cursor()#执行SQL命令并提交到数据库执行#execute()方法第二个参数为列表传参补位

cursor.execute('insert into filmtab values(%s,%s,%s)',['霸王别姬','张国荣','1993'])

db.commit()#关闭

cursor.close()

db.close()

3.来试试高效的executemany()方法？

importpymysql#创建2个对象

db = pymysql.connect('192.168.153.137','tiger','123456','maoyandb',charset='utf8')

cursor=db.cursor()#抓取的数据

film_list = [('月光宝盒','周星驰','1994'),('大圣娶亲','周星驰','1994')]#执行SQL命令并提交到数据库执行#execute()方法第二个参数为列表传参补位

cursor.executemany('insert into filmtab values(%s,%s,%s)',film_list)

db.commit()#关闭

cursor.close()

db.close()

importpymysql#__init__(self)：

self.db = pymysql.connect('IP',... ...)

self.cursor=self.db.cursor()#write_data(self):

self.cursor.execute('sql',[data1])

self.cursor.executemany('sql',[(data1),(data2),(data3)])

self.db.commit()#main(self):

self.cursor.close()

self.db.close()

4.将电影信息存入MySQL数据库(尽量使用executemany方法)

importpymysqlfrom urllib importrequest, parseimportreimporttimeimportrandomfrom useragents importua_listclassMaoyanSpider(object):def __init__(self):

self.url= 'https://maoyan.com/board/4?offset={}'

#计数

self.num =0

self.db=pymysql.connect('localhost', 'root', '123456', 'maoyandb', charset='utf8')

self.cursor=self.db.cursor()defget_html(self, url):

headers={'User-Agent': random.choice(ua_list)

}

req= request.Request(url=url, headers=headers)

res=request.urlopen(req)

html= res.read().decode('utf-8')#直接调用解析函数

self.parse_html(html)defparse_html(self, html):#创建正则的编译对象

re_ = '