# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import requests
import json
import time
import pymysql
import importlib,sys
importlib.reload(sys)forum_note_sum=[]  #variavle：save the content of tiezi
list_d=['原创','翻译','讨论']  #内容判断条件，如果帖子标题内容为此，取另一个值
type = sys.getfilesystemencoding()
#num:the record number of one page；get tiezi of author and others
def parent_li_web(num):forum_note_record = {}try:parent_tiezi=bs_obj.find('ul',class_='for-list').find_all('li')[num]div_one = parent_tiezi.find('div', class_='titlelink box')div_two = parent_tiezi.find('div', class_='author box')span_three = parent_tiezi.find('span', class_='ansour box').string.strip()div_four = parent_tiezi.find('div', class_='endreply box')subname=div_one.a.stringsublink='https://bbs.hupu.com'+div_one.a['href']team_tmp=theme_tmpfor i in list_d:if i==subname:subname=div_one.find_all('a')[1].stringsublink='https://bbs.hupu.com'+div_one.find_all('a')[1]['href']# print (i,subname,sublink)forum_note_record.update({'subname':subname,'subname_link':sublink,'author':div_two.a.string,'author_link':div_two.a['href'],'author_create_time':div_two.find('a',style='color:#808080;cursor: initial; ').string,'read_reply_number':span_three,'last_reply_writer':div_four.span.string,'last_reply_time':div_four.a.string,'team_tmp':team_tmp})forum_note_sum.append(forum_note_record)except:return Noneif __name__=='__main__':# all_spurs_notebegin_time=time.time()print('---------脚本执行时间为：{}------------'.format(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime())))team_list = ['rockets', 'warriors', 'cavaliers', 'spurs', 'lakers', 'celtics', 'thunder', 'clippers','timberwolves', 'mavericks', 'knicks', 'bulls', 'nets', 'sixers', 'jazz', 'pacers', 'blazers', 'heat','suns', 'grizzlies', 'wizards', 'pelicans', 'bucks', 'kings', 'raptors', 'nuggets', 'hawks', 'hornets','pistons', 'magic']for li in team_list:forum_note_sum_code=[]theme_tmp=lifor i in range(1,11,1):   #由于虎扑反爬，只能爬到10页；后续可放入linux中定时执行url = 'https://bbs.hupu.com/{}-{}'.format(li,i)print (url)wb_string = requests.get(url)bs_obj = BeautifulSoup(wb_string.content, 'html.parser')with open('web_spider_original.txt','w',encoding='utf8') as f:f.write(str(bs_obj))f.write('\r'*10+'-----我是分割线-----'+'\r'*10)for j in range(1,61,1):   #每个页面数据有60个帖子parent_li_web(j)with open('hupu_spider_spurs_load.txt', 'w', encoding='utf8') as f:for item in forum_note_sum:json.dump(item,f,ensure_ascii=False)f.write('\r')#insert into mysqlconn=pymysql.connect(host='localhost',user='root',passwd='1234',db='spider',port=3306,charset='utf8')cur=conn.cursor()cur.execute('delete from hupuforum_spurs_note_daytmp')with open('hupu_spider_spurs_load.txt','r',encoding='utf8') as f:for item in f:item=json.loads(item) #how convert string to dict# print(type(item))cur.execute('insert into hupuforum_spurs_note_daytmp(subname,subname_link,author,author_link,author_create_time,read_reply_number,last_reply_writer,last_reply_time,theme_title) values(%s,%s,%s,%s,%s,%s,%s,%s,%s)',(item['subname'],item['subname_link'],item['author'],item['author_link'],item['author_create_time'],item['read_reply_number'],item['last_reply_writer'],item['last_reply_time'],item['team_tmp']))conn.commit()cur.close()conn.close()print('Finished!本次执行消耗时间为：{}秒'.format(time.time()-begin_time))

# coding=utf8
import time
import requests
from bs4 import BeautifulSoup
import pymysql
import signalbegin_time=time.strftime('%Y-%m-%d %H:%M:%S',time.localtime())
conn=pymysql.connect(host='localhost',port=3306,user='root',passwd='1234',db='spider',charset='utf8')
cur=conn.cursor()
sub_cur = conn.cursor()
cur.execute('INSERT INTO hupuforum_spurs_note SELECT * FROM hupuforum_spurs_note_daytmp WHERE subname_link NOT IN (SELECT a.subname_link FROM hupuforum_spurs_note a);')
cur.execute('update hupuforum_spurs_note a,hupuforum_spurs_note_daytmp b set a.read_reply_number=b.read_reply_number,a.last_reply_writer=b.last_reply_writer,a.last_reply_time=b.last_reply_time where a.subname_link=b.subname_link ')
# conn.commit()
cur.execute('use spider;')
conn.commit()
cur.execute('select subname_link from hupuforum_spurs_note where sub_text is null;')
for url in cur.fetchall():url=list(url)# print(url)try:wb_page = requests.get(url[0],timeout=2)   #实际执行中，存在网页假死状态，设置超时bs_obj = BeautifulSoup(wb_page.content, 'html.parser')tmp_text = bs_obj.select('#tpc > div > div.floor_box > table.case > tbody > tr > td > div.quote-content')sub_text=tmp_text[0].get_text(strip=True)sub_text=sub_text.replace('\'','’')sql="""update hupuforum_spurs_note set sub_text=\'{}\' where subname_link={};""".format((sub_text[:1000]),str(url).replace('[','').replace(']',''))# print(sql)
        sub_cur.execute(sql)conn.commit()print('success')except IndexError as e:   #这个错误意味着页面也不存在sql="""update hupuforum_spurs_note set sub_text=\'{}\' where subname_link={};""".format('网页不存在',str(url).replace('[','').replace(']',''))sub_cur.execute(sql)conn.commit()except pymysql.err.InternalError as e:  #说明内容中包含emoj等utf8四字符内容sql="""update hupuforum_spurs_note set sub_text=\'{}\' where subname_link={};""".format('内容格式有误，导致出错！',str(url).replace('[','').replace(']',''))sub_cur.execute(sql)conn.commit()except requests.exceptions.ReadTimeout as e: #网页响应超时sql="""update hupuforum_spurs_note set sub_text=\'{}\' where subname_link={};""".format('网页打开超时',str(url).replace('[','').replace(']',''))sub_cur.execute(sql)conn.commit()else:sql="""update hupuforum_spurs_note set sub_text=\'{}\' where subname_link={};""".format('其他类型错误',str(url).replace('[','').replace(']',''))sub_cur.execute(sql)conn.commit()
conn.commit()
cur.close()
sub_cur.close()
conn.close()
end_time=time.strftime('%Y-%m-%d %H:%M:%S',time.localtime())
print('Finished，任务开始时间为：{}，结束时间为：{}'.format(begin_time,end_time))

# coding=utf8
import time
import requests
from bs4 import BeautifulSoup
import pymysqlbegin_time=time.strftime('%Y-%m-%d %H:%M:%S',time.localtime())
conn=pymysql.connect(host='localhost',port=3306,user='root',passwd='1234',db='spider',charset='utf8')
cur=conn.cursor()
sub_cur=conn.cursor()
cur.execute('select distinct author_link from hupuforum_spurs_note;')
for author_url in cur.fetchall():try:author_url=list(author_url)wb_obj=requests.get(author_url[0],timeout=2)bs_obj=BeautifulSoup(wb_obj.content,'html.parser')author=bs_obj.select('#main > div.personal > div.personal_right > h3 > div')[0].stringauthor_visited=bs_obj.select('#main > div.personal > div.personal_right > h3 > span')[0].string.replace('有','').replace('人次访问','')author_info=bs_obj.select('#main > div.personal > div.personal_right > div')[0].get_text(strip=True)sub_cur.execute('insert into hupuforum_authors_info(author,author_link,author_visited,author_info,author_status) values(%s,%s,%s,%s,%s)',(author,author_url[0],author_visited,author_info,'正常'))except IndexError as e:sub_cur.execute('insert into hupuforum_authors_info(author,author_link,author_visited,author_info,author_status) values(%s,%s,%s,%s,%s)',(author, author_url[0], '', '', '无法访问'))conn.commit()
conn.commit()
cur.close()
conn.close()
end_time=time.strftime('%Y-%m-%d %H:%M:%S',time.localtime())
print('Finished，任务开始时间为：{}，结束时间为：{}'.format(begin_time,end_time))