mysql - u root -p xxx <weibo500w.sql

        去重复内容：delete from 表名 where id not in (select minid from (select min(id) as minid from 表名 group by 字段) b);
去值为NULL：delete from 表名 where 字段名=NULL
去值为""空值delete from 表名 where 字段名=''

        import re
from snownlp import sentiment
import numpy as np
import pymysql
from snownlp import SnowNLP
import matplotlib.pyplot as plt
from snownlp import sentiment
from snownlp.sentiment import Sentiment
conn = pymysql.connect(host='数据库IP', user='用户名', password='密码', charset="utf8",use_unicode=False)  # 连接服务器
with conn:cur = conn.cursor()cur.execute("SELECT * FROM test.weibo WHERE weiboId < '%d'" % 6000000)rows = cur.fetchall()
comment = []
for row in rows:row = list(row)comment.append(row[18])
def train_model(texts):for li in texts:comm = li.decode('utf-8')text = re.sub(r'(?:回复)?(?://)?@[\w\u2E80-\u9FFF]+:?|\[\w+\]', ',',comm)socre = SnowNLP(text)if socre.sentiments > 0.8:with open('pos.txt', mode='a', encoding='utf-8') as g:g.writelines(comm +"\n")elif socre.sentiments < 0.3:with open('neg.txt', mode='a', encoding='utf-8') as f:f.writelines(comm + "\n")else:passtrain_model(comment)
sentiment.train('neg.txt', 'pos.txt')
sentiment.save('sentiment.marshal')

        {"comments": [{"created_at": "Wed Jun 01 00:50:25 +0800 2011","id": 12438492184,"text": "love your work.......","source": "<a href="http://weibo.com" rel="nofollow">新浪微博</a>","mid": "202110601896455629","user": {"id": 1404376560,"screen_name": "zaku","name": "zaku","province": "11","city": "5","location": "北京 朝阳区","description": "人生五十年，乃如梦如幻；有生斯有死，壮士复何憾。","url": "http://blog.sina.com.cn/zaku","profile_image_url": "http://tp1.sinaimg.cn/1404376560/50/0/1","domain": "zaku","gender": "m","followers_count": 1204,"friends_count": 447,"statuses_count": 2908,"favourites_count": 0,"created_at": "Fri Aug 28 00:00:00 +0800 2009","following": false,"allow_all_act_msg": false,"remark": "","geo_enabled": true,"verified": false,"allow_all_comment": true,"avatar_large": "http://tp1.sinaimg.cn/1404376560/180/0/1","verified_reason": "","follow_me": false,"online_status": 0,"bi_followers_count": 215},}},...],"previous_cursor": 0,"next_cursor": 0,"total_number": 7
}

        import codecs
import re
import numpy as np
import pymysql
from snownlp import SnowNLP
import matplotlib.pyplot as plt
from snownlp import sentiment
from snownlp.sentiment import Sentimentcomment = []
with open('文件路径', mode='r', encoding='utf-8') as f:rows = f.readlines()for row in rows:if row not in comment:comment.append(row.strip('\n'))
def snowanalysis(self):sentimentslist = []for li in self:#text = re.sub(r'(?:回复)?(?://)?@[\w\u2E80-\u9FFF]+:?|\[\w+\]', ',',li)print(li)s = SnowNLP(li)print(s.sentiments)sentimentslist.append(s.sentiments)plt.hist(sentimentslist, bins=np.arange(0, 1, 0.01))plt.show()
snowanalysis(comment)

        import pickle
from os import path
import jieba
import matplotlib.pyplot as plt
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
def make_worldcloud(file_path):text_from_file_with_apath = open(file_path,'r',encoding='UTF-8').read()wordlist_after_jieba = jieba.cut(text_from_file_with_apath, cut_all=False)wl_space_split = " ".join(wordlist_after_jieba)print(wl_space_split)backgroud_Image = plt.imread('图片路径')print('加载图片成功！')'''设置词云样式'''stopwords = STOPWORDS.copy()stopwords.add("哈哈") #可以加多个屏蔽词wc = WordCloud(width=1024,height=768,background_color='white',# 设置背景颜色mask=backgroud_Image,# 设置背景图片font_path='E:\simsun.ttf',  # 设置中文字体，若是有中文的话，这句代码必须添加，不然会出现方框，不出现汉字max_words=600, # 设置最大现实的字数stopwords=stopwords,# 设置停用词max_font_size=400,# 设置字体最大值random_state=50,# 设置有多少种随机生成状态，即有多少种配色方案)wc.generate_from_text(wl_space_split)#开始加载文本img_colors = ImageColorGenerator(backgroud_Image)wc.recolor(color_func=img_colors)#字体颜色为背景图片的颜色plt.imshow(wc)# 显示词云图plt.axis('off')# 是否显示x轴、y轴下标plt.show()#显示# 获得模块所在的路径的d = path.dirname(__file__)# os.path.join()：  将多个路径组合后返回wc.to_file(path.join(d, "h11.jpg"))print('生成词云成功!')make_worldcloud('文本路径')