[Python] 纯文本查看 复制代码# -*- coding:utf-8 -*-

import os

import urllib.parse

from html.parser import HTMLParser

import requests

from bs4 import BeautifulSoup

from pymongo import MongoClient

class ContentHtmlParser(HTMLParser):

"""

过滤html标签

"""

def __init__(self):

HTMLParser.__init__(self)

self.text = ""

def handle_data(self, data):

self.text += data

def get_text(self):

return self.text

mongo_client = MongoClient("ip", 27017)

mongo_db = mongo_client["gongzhonghao"]

def get_words():

words = []

with open("words.txt", encoding="utf-8") as words_file:

for lines in words_file.readlines():

if len(lines.strip()) == 0:

continue

if lines.find("、") != -1:

for p in lines.split("、"):

words.append(p.replace("\n", ""))

else:

words.append(lines.replace("\n", ""))

return words

def get_articles(clt):

articles = []

collection = mongo_db[clt]

doc = collection.find_one()

items = doc["items"]

for it in items:

content = it["content"]["news_item"][0]

articles.append(content)

return articles

def download(dir, file_name, url):

if not os.path.exists(dir):

os.mkdir(dir)

try:

resp = requests.get(url)

path = dir + "\\" + file_name

if os.path.exists(path):

return

with open(path, "wb") as f:

f.write(resp.content)

except :

print(url)

def find_images(content):

imgs = []

c = urllib.parse.unquote(content)

img_labels = BeautifulSoup(c, "html.parser").find_all("img")

for img in img_labels:

src = img.get("data-src")

imgs.append(src)

return imgs

def get_suffix(url):

try:

suffix = url[url.rindex("=") + 1:]

if suffix == "jpeg" or suffix == "other":

return ".jpg"

return "." + suffix

except:

return ".jpg"

def filter_content(content):

parser = ContentHtmlParser()

parser.feed(content)

return parser.get_text()

def check_jinyongci(content):

fc = filter_content(content)

words = get_words()

invalids = []

for w in words:

if fc.find(w) != -1:

invalids.append(w)

return invalids

def save_jinyongci(clt, title, invalids):

if len(invalids) == 0:

return

file = clt + "\\invalid.txt"

with open(file, "a+",encoding="utf-8") as f:

f.write("标题：" + title)

f.write("\r\n敏感词：")

for iv in invalids:

f.write(iv)

f.write("、")

f.write("\r\n\r\n")

if __name__ == "__main__":

clt = "xxx"

if not os.path.exists(clt):

os.mkdir(clt)

articles = get_articles(clt)

print(clt + ": 共" + str(len(articles)) + "个")

for i in range(0, len(articles)):

print("正在处理第 " + str(i) + " 个")

title = articles[i]["title"]

thumb_url = articles[i]["thumb_url"]

content = articles[i]["content"]

# 下载封面

# path = os.path.join(clt, title)

fname = str(i) + "_" + title.replace("|", "").replace("", "")

download(clt, fname + get_suffix(thumb_url), thumb_url)

# 找出文章中的图片

imgs = find_images(content)

index = 0

for img in imgs:

download(clt, fname + "_" + str(index) + get_suffix(img), img)

index = index + 1

# 找出文章中的敏感词

invalids = check_jinyongci(content)

print(invalids,'----',title)

save_jinyongci(clt, title, invalids)