#-*- coding = utf-8 -*-

#@Time : 2020/7/23 15:09

#@Author :

#@File : douban_group.py

#@software : PyCharm

#爬取豆瓣组数

#待解决的问题：标签下小组不全！！

import csv

import importlib

import urllib.request

import urllib.error

import re

from random import sample

from urllib.parse import quote

import string

import regex

from bs4 import BeautifulSoup

import sqlite3

import xlwt

import pandas as pd

#确定一个主函数

def main():

baseurl = "https://www.douban.com/"

datalist = getDATA(baseurl)

#print(datalist)

pd.DataFrame(datalist).to_csv("豆瓣小组.csv")

#savepath = "豆瓣组员数超过一万的小组.xls"

#saveDATA(datalist)

findtag = regex.compile(r'

(.*)')

findlink = regex.compile('

')

def getDATA(url):

#得到豆瓣首页

global pagenum

datalist = []

html = askURL(url)

# 对得到的首页进行信息筛选

soup = BeautifulSoup(html, "html.parser")

for cate in soup.find_all('div', class_="cate group-cate"):#筛选到tag和link

cate = str(cate)

#link = regex.findall(findlink, cate, overlapped=True)

link = "https://www.douban.com/group/explore?"

tag = regex.findall(findtag, cate, overlapped=True)

#print(tag)

findsum = re.compile(r'

')

findname = re.compile(r'')

findpagenum = re.compile(r'1')

# 首先分别把各tag中的总页数筛出来

for i in range(len(tag)):

url1 = link + "tag=" + quote(str(tag[i]))

html_temp = askURL(url1)

soup_temp = BeautifulSoup(html_temp, "html.parser")

for paginator in soup_temp.find_all('div',class_="paginator"):

paginator = str(paginator)

pagenum = re.findall(findpagenum, paginator)[0]

# 重新进入link，获取soup

for j in range(0, int(pagenum)):

url2 = link + "start=" + str(j*20) + "&tag=" + quote(str(tag[i]))

html_temp1 = askURL(url2)

soup_temp1 = BeautifulSoup(html_temp1, "html.parser")

for result in soup_temp1.find_all('div', class_="result"):

data = {}

result = str(result)

name =re.findall(findname, result)[0]

data['name'] = name

sum = re.findall(findsum, result)[0]

data['sum'] = sum

datalist.append(data)

return datalist

my_headers = [

"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",

"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",

"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",

"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",

"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",

'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',

'Opera/9.25 (Windows NT 5.1; U; en)',

'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',

'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',

'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',

'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',

"Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",

"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0 "

]

def askURL(url):

#模拟浏览器头部文件

head = {

"Host": " www.douban.com",

"User-Agent": sample(my_headers, 1)[0]}

req = urllib.request.Request(url=url, headers=head)

html = ""

try:

response = urllib.request.urlopen(req)

html = response.read().decode("utf-8")

except urllib.error.URLError as e:

if hasattr(e, "reason"):

print(e.reason)

return html

#def saveDATA(datalist):

# pf = pd.DataFrame(list(datalist))

# # 指定字段顺序

# order = ['name', 'sum']

# pf = pf[order]

# # 将列名替换为中文

# columns_map = {

# 'name': '豆瓣小组',

# 'sum': '组员数'

# }

# pf.rename(columns=columns_map, inplace=True)

# # 指定生成的Excel表格名称

# file_path = pd.ExcelWriter('doubangroup.xlsx')

# # 替换空单元格

# pf.fillna(' ', inplace=True)

# # 输出

# pf.to_excel(file_path, encoding='utf-8', index=False)

# # 保存表格

# file_path.save()

if __name__ == "__main__":

main()

print("爬取成功")