python 爬虫爬取bilibili

selenium

**
Selenium 是一个用于Web应用程序测试的工具。
使用说明
https://selenium-python.readthedocs.io/
下载谷歌驱动器chromedriver
https://npm.taobao.org/mirrors/chromedriver/
找到你谷歌浏览器对应的版本的驱动器，并将驱动器放在谷歌浏览器所在的位置

连接数据库

用法

利用selenium包中的函数主要有
driver.get(website)打开website连接
driver.find_elements_by_xpath(xpath)通过xpath查找网页中对应的元素
link.get_attribute(‘href’) 获得属性的值，这里为获得链接

利用XPath的主要方法
https://www.runoob.com/xpath/xpath-tutorial.html
// 从匹配选择的当前节点选择文档中的节点，而不考虑它们的位置
/ 从根节点选取
@ 选取属性
*匹配任何元素节点。

爬虫爬取Bilibili

思路：
先爬取导航栏的各话题

然后再打开这些链接，爬取每个链接推荐的视频链接
比如动画话题

爬取视频所有导航栏话题网页推荐的视频并放入队列，再利用广度优先算法，每次取队头，并提取视频的元素，并插入到数据库，之后爬取每个推荐的视频链接并放队尾，这样相当于每个话题都一层一层的爬取，没有终止。

我爬取了视频的链接，标题，作者，播放量，弹幕数，相关，发布时间。
这里我导航栏话题爬取下面的链接没有写完，可以通过获得每个类型的路径，用XPath = "路径1 | 路径2 | …"爬取完整
| 相当于或
爬取结果

代码：

from datetime import datetime
import dateutil.parser
from furl import furl
from pymongo import MongoClient
from selenium import webdriver
from selenium.common.exceptions import *
from urlextract import URLExtract
from queue import Queue# 数据库
DATABASE_URL = 'mongodb://localhost:27017/'
DATABASE_NAME = 'Bilibili'
COLLECTION_NAME = 'Bilibili'# 数据库连接
def connect_database():client = MongoClient(DATABASE_URL)database = client[DATABASE_NAME]return client, database# 检查是否存在该数据,防止重复插入
def check_database(url, database):collection = database[COLLECTION_NAME]record = collection.find_one({'url': url})return record is not None# 写入数据库
def write_database(url,  # 视频链接title,  # 视频名author,  # 作者created_time,  # 发布时间Play_volume,  # 播放量barrages,  # 弹幕数related,  # 相关database):collection = database[COLLECTION_NAME]# 集合record = {'url': url,'title': title,'author': author,'created_time': created_time,'Play_volume': Play_volume,'barrages': barrages,'related': related}# 打印集合print(record)# 插入数据record_id = collection.insert_one(record).inserted_idreturn record_id# 获得所有话题链接并排除直播，番剧链接
def get_top_pages(driver, main_topic_page):driver.get(main_topic_page)# 此处需要考虑到一般的导航栏项目和More选项中的项目xpath = "//div[@class='primary-menu-itnl']/div[@id='primaryChannelMenu']//a | //div[@class='sub-item']/a"subtopic_pages = driver.find_elements_by_xpath(xpath)subtopic_pages = [x.get_attribute('href') for x in subtopic_pages]subtopic_pages = [furl(x).remove(args=True, fragment=True).url for x in subtopic_pages]# 去重subtopic_pages = list(set(subtopic_pages))return subtopic_pages# 进入话题子网站，收集所有视频链接
def get_sub_link(driver, subtopic_pages):# 判断是否为网址if subtopic_pages.startswith("https"):passelse:news_pages = []return news_pages# 爬取链接driver.get(subtopic_pages)xpath = "//div[@class='channel-m']//a"try:sub_link = driver.find_elements_by_xpath(xpath)# 过滤掉不可见的元素news_pages = [x for x in sub_link if x.is_displayed()]print('-' * 50)print('sub_link page:', subtopic_pages)for each in news_pages:print(each.text, each.get_attribute('href'))news_pages = [each.get_attribute('href') for each in news_pages]news_pages = list(set(news_pages))except NoSuchElementException:news_pages = []return news_pages# 获得该视频作为的视频链接
def get_around_website(driver, headwebsite):# 判断是否为网址if headwebsite.startswith("https"):passelse:pages = []return pagesdriver.get(headwebsite)xpath = "//div[ @class ='rec-list']//div[@ class ='pic']/a"around_website = driver.find_elements_by_xpath(xpath)pages = [x for x in around_website if x.is_displayed()]pages = [each.get_attribute('href') for each in around_website]pages = list(set(pages))return pages# 广度优先算法获取相关视频
def breadth_first(driver, webside, database):q = Queue(maxsize=-1)# 放入种子网址for w in webside:q.put(w)while (~q.empty()):# 取队头headwebsite = q.get()# 读取队头网址的内容retry_num = 3try:while retry_num > 0:try:get_news_content(driver, headwebsite, database)breakexcept Exception as e:retry_num -= 1except Exception as e:print(e)print('Max retry num reached')continue# 获得与其相关的视频链接around_website = get_around_website(driver, headwebsite)# 放入队尾for aw in around_website:q.put(aw)# 截取该视频的信息
def get_news_content(driver, news_page, database):# 去除各种参数后的URLurl = furl(news_page).remove(args=True, fragment=True).urldriver.get(news_page)try:title_xpath = "//h1[@class='video-title']/span"title = driver.find_element_by_xpath(title_xpath).textexcept NoSuchElementException:print('no title')return# 作者（不存在则填写媒体名称）author_xpath = "//div[@class='up-info_right']/div[@class='name']/a"try:author = driver.find_element_by_xpath(author_xpath).textexcept NoSuchElementException:author = None# 发布时间（字符串和秒级时间戳）try:created_time_xpath = "//div[@class='video-data']/span[3]"created_time = driver.find_element_by_xpath(created_time_xpath).textexcept NoSuchElementException:print('no created time')returncreated_time = dateutil.parser.parse(created_time)created_time = created_time.strftime('%Y/%m/%d %H:%M')# 播放量try:play_volume_xpath = "//div[@class='video-data']/span[1]"play_volume = driver.find_element_by_xpath(play_volume_xpath).textexcept NoSuchElementException:print('no created time')returnplay_volume = play_volume.replace("播放", " ")play_volume = play_volume.replace("·", "")# 弹幕数try:barrages_xpath = "//div[@class='video-data']/span[2]"barrages = driver.find_element_by_xpath(barrages_xpath).textexcept NoSuchElementException:print('no created time')returnbarrages = barrages.replace("弹幕", " ")# 相关话题try:related_xpath = "//li[@class='tag']/div/a/span"related = driver.find_elements_by_xpath(related_xpath)related = [each.text for each in related]except NoSuchElementException:print('no created time')return# 打印爬取的信息print("视频链接" + url + "\n标题" + title + "\n作者" + author + "\n发布时间" + created_time + "\n播放量" + play_volume + "\n弹幕数" + barrages + "\n相关")for r in related:print(r + " ", end='')write_database(url,  # 视频链接title,  # 视频名author,  # 作者created_time,  # 发布时间play_volume,  # 播放量barrages,  # 弹幕数related,  # 相关database)def main():from selenium.webdriver.chrome.options import Optionsclient, database = connect_database()options = Options()prefs = {"profile.managed_default_content_settings.images": 2}options.add_experimental_option("prefs", prefs)options.add_argument('--headless')driver = webdriver.Chrome(options=options)topic_page = "https://www.bilibili.com/"# 开始爬取# 获得导航栏话题subtopic_link = get_top_pages(driver, topic_page)# 进入导航栏爬取主页面的所有视频链接newlink = []for link in subtopic_link:each = get_sub_link(driver, link)newlink.extend(each)# 广度优先算法breadth_first(driver, newlink, database)driver.close()driver.quit()if __name__ == '__main__':import timet1 = time.time()main()t2 = time.time()print(t2 - t1)