#对文本进行分词
def get_divide(i):filename = 'file' + str(i + 1) + '.txt'stop_words = set(line.strip() for line in open('stopwords.txt',encoding = 'utf-8'))"""
这里要用只读模式打开，否者会将上一个方法写入的内容覆盖掉。
另外要注意用二进制模式（ rb(二进制读) , wb(二进制写)）打开，否者会编码错误：
UnicodeDecodeError: 'gbk' codec can't decode byte 0x90 in position 2: illegal multibyte sequence"""with open(filename,'rb') as f:file = f.readlines()for each in file:"""直接jieba.cut()方法会报错：not enough values to unpack (expected 2, got 1) 先引入jieba.posseg模块就不会。"""seg_list = pseg.cut(each)#flag是用于判断词性的for item,flag in seg_list:               if not item in stop_words and flag == 'n':print(item)

import requests
from bs4 import BeautifulSoup
import re
import time
import jieba
import jieba.posseg as psegdef getReq(i):try:url_1 = "http://www.gaoloumi.com/forum.php?mod=forumdisplay&fid=30&page="url = url_1 + str(i)r = requests.get(url)r.raise_for_statusr.encoding = r.apparent_encodingreturn r.textexcept:print('')#获取html文本
def getHtml(r):soup = BeautifulSoup(r,'lxml')html = soup.find_all('a','s xst')return html#将每个页面所有帖子的名称写入文件
def writeHtml(html,i):try:fileName = 'file' + str( i + 1 ) + '.txt'#注意编码格式with open(fileName,'w+',encoding = 'utf-8') as f:for each in html:f.write(each.string)f.write('\n')except:pass#对文本进行分词
def get_divide(i):filename = 'file' + str(i + 1) + '.txt'stop_words = set(line.strip() for line in open('stopwords.txt',encoding = 'utf-8'))"""
这里要用只读模式打开，否者会将上一个方法写入的内容覆盖掉。
另外要注意用二进制模式（ rb(二进制读) , wb(二进制写)）打开，否者会编码错误：
UnicodeDecodeError: 'gbk' codec can't decode byte 0x90 in position 2: illegal multibyte sequence"""with open(filename,'rb') as f:file = f.readlines()for each in file:"""直接jieba.cut()方法会报错：not enough values to unpack (expected 2, got 1) 先引入jieba.posseg模块就不会。"""seg_list = pseg.cut(each)#flag是用于判断词性的for item,flag in seg_list:               if not item in stop_words and flag == 'n':print(item)#主函数
def main():for i in range(5):r = getReq(i)html = getHtml(r)writeHtml(html,i)get_divide(i)print("第" + str(i+1) + "个网页爬取成功！")time.sleep(1)
main()