')

#items = re.findall(pattern, html.text)

#print(html.text)

#使用json格式化输出

json_obj =html.text

j=json.loads(json_obj)

i=j['hotComments']for uu ini:printusername=uu["user"]['nickname']

likedCount1= str(uu['likedCount'])

comments=uu['content']

with open(name+ "的热评hotComment" +".txt" , 'a+',encoding='utf8') as f:

f.write("用户名是"+username+"\n")

f.write("用户的评论是"+comments+"\n")

f.write("被点赞的次数是" + str(likedCount1) +"\n")

f.write("----------华丽的的分割线-------------"+"\n")

f.close()

上面需要注意的是：利用json获取需要的数据(至少比正则快点)

结果如下:

到这里！！我们已经完成了所有用数据的爬取，数据大体如下

三 数据分析，可视化

如果数据不利用，就如同一张白纸一样，毫无意义。接下来我们就对数据进行全面的分析

第一步，我们先进行歌词的分析，先把数据合并到一个文件里

defMergedFile(self):

aaa=0for i in glob.glob("*歌曲名*"):

file_object= open(i,'r',encoding='UTF-8')

list_of_line=file_object.readlines()for p inlist_of_line:if "作词" in p or "作曲" in p or "混音助理" in p or "混音师" in p or "录音师" in p or "执行制作" in p or "编曲" in p or "制作人" in p or "录音工程" in p or "录音室" in p or "混音录音室" in p or "混音工程" in p or "Programmer" in p or p == "\n" or "和声" in p or "吉他" in p or "录音助理" in p or "陈任佑鼓" in p or "周杰伦" inp:

aaa+=1

print(p)else:

with open ("allLyric"+".txt","a",encoding='UTF-8') as f :

f.write(p)

f.write("\n")print(aaa)#合并歌曲

file1 = open('allLyric.txt', 'r', encoding='utf-8') #要去掉空行的文件

file2 = open('allLyric1.txt', 'w', encoding='utf-8') #生成没有空行的文件

try:for line infile1.readlines():if line == '\n':

line= line.strip("\n")

file2.write(line)finally:

file1.close()

file2.close()print("合并歌词文件完成")

上面需要注意的是：我们合并数据的时候，可以选择性的删除一些无用数据(也就是上面那一大串for,麻瓜式的删除)

结果如下

OK 歌词清洗完成之后，我们对周杰伦歌曲进行情绪化分析，在进行情绪化分析的时候，有很多工具可以选择，我这边选择的是SnowNLP这个三方工具，具体代码如下:

defEmotionAnalysis(self):from snownlp importSnowNLPfrom pyecharts importBar

xzhou=[]

yzhou=[]for i in glob.glob("*歌曲名*"):

count=0

allsen=0

with open(i,'r', encoding='utf-8') as fileHandel:

fileList=fileHandel.readlines()for p infileList:if "作词" in p or "作曲" in p or "鼓" in p or "混音师" in p or "录音师" in p or "执行制作" in p or "编曲" in p or "制作人" in p or "录音工程" in p or "录音室" in p or "混音录音室" in p or "混音工程" in p or "Programmer" in p or p == "\n":pass

else:

s=SnowNLP(p)#print(s.sentences[0])

s1 =SnowNLP(s.sentences[0])#print(type(s1))

count+=1allsen+=s1.sentiments

i=str(i)

xzhou1= i.split("-", 1)[1].split(".",1)[0]

xzhou.append(xzhou1)

avg=int(allsen)/count

yzhou.append(avg)#print("%s这首歌的情绪为%s"%(i,avg))

fileHandel.close()

bar= Bar("柱状图数据堆叠示例")

bar.add("周杰伦歌曲情绪可视化", xzhou, yzhou, is_stack=True,xaxis_interval=0)

bar.render(r"D:\学习\untitled4\allpicture\周杰伦歌曲情绪全部.html")#显示最好的前五首歌

importheapq

yzhou1= heapq.nlargest(10, yzhou)

temp= map(yzhou.index, heapq.nlargest(10, yzhou))

temp=list(temp)

xzhou1=[]for i intemp:

xzhou1.append(xzhou[i])#情绪前十首歌个图

bar = Bar("周杰伦歌曲情绪较好前十首歌")

bar.add("周杰伦歌曲情绪可视化", xzhou1, yzhou1, is_stack=True)

bar.render(r"D:\学习\untitled4\allpicture\周杰伦歌曲最积极情绪top10.html")#显示最差的十首歌

yzhou1 = heapq.nsmallest(10, yzhou)

temp= map(yzhou.index, heapq.nsmallest(10, yzhou))

temp=list(temp)

xzhou1=[]for i intemp:

xzhou1.append(xzhou[i])#print(xzhou1)

#print(yzhou1)

#情绪前十首歌个图

bar = Bar("周杰伦歌曲情绪较差前十首歌")

bar.add("周杰伦歌曲情绪可视化",xzhou1, yzhou1,xaxis_interval=0,xzhou1_label_textsize=6)

bar.render(r"D:\学习\untitled4\allpicture\周杰伦歌曲最消极情绪top10.html")print(xzhou1)

下面完成数据词频各种分析,这些基于数据source的统计用很多方法，我这边做为小白只能用比较傻瓜一点的，具体代码如下：

#定义结巴分词的方法以及处理过程

defsplitSentence(self,inputFile, outputFile):

fin= open(inputFile, 'r', encoding='utf-8')

fout= open(outputFile, 'w', encoding='utf-8')for line infin:

line=line.strip()

line=jieba.analyse.extract_tags(line)

outstr= " ".join(line)

fout.write(outstr+ '\n')

fin.close()

fout.close()#下面的程序完成分析前十的数据出现的次数

f = open("分词过滤后.txt", 'r', encoding='utf-8')

a=f.read().split()

b= sorted([(x, a.count(x)) for x in set(a)], key=lambda x: x[1], reverse=True)print(sorted([(x, a.count(x)) for x in set(a)], key=lambda x: x[1], reverse=True))#输出频率最多的前十个字，里面调用splitSentence完成频率出现最多的前十个词的分析

defLyricAnalysis(self):importjieba

file= 'allLyric1.txt'

#这个技巧需要注意

alllyric = str([line.strip() for line in open('allLyric1.txt',encoding="utf-8").readlines()])#获取全部歌词，在一行里面

alllyric1=alllyric.replace("'","").replace(" ","").replace("?","").replace(",","").replace('"','').replace("?","").replace(".","").replace("!","").replace(":","")#print(alllyric1)

#在这里用结巴分词来分词过滤并且输出到一个文件里面，这个ting.txt

#import jieba.analyse 这里必须引入

jieba.analyse.set_stop_words("ting.txt")

self.splitSentence('allLyric1.txt', '分词过滤后.txt')#下面是词频统计

importcollections#读取文本文件，把所有的汉字拆成一个list

f = open("分词过滤后.txt", 'r', encoding='utf8') #打开文件，并读取要处理的大段文字

txt1 =f.read()

txt1= txt1.replace('\n', '') #删掉换行符

txt1 = txt1.replace(' ', '') #删掉换行符

txt1 = txt1.replace('.', '') #删掉逗号

txt1 = txt1.replace('.', '') #删掉句号

txt1 = txt1.replace('o', '') #删掉句号

mylist =list(txt1)

mycount=collections.Counter(mylist)for key, val in mycount.most_common(10): #有序(返回前10个)

print(key, val)

接下来我们需要进行天气，词云化数据，比如这里的天气我选择了用shell完成统计。-----cat XXXX | grep 秋天 | wc -l  这样一句话就OK 了，没必要再使用python3 物尽其用，方得正果：这些可视化的代码都很类似，就不全部列出来了。放一个示例吧：

from pyecharts importWordCloud

name=["离开", "回忆", "微笑", "爱情", "世界", "想要", "眼泪", "不到", "感觉", "我会", "喜欢", "时间", "Oh", "永远", "oh", "记得", "不用", "真的", "一种","我用", "等待", "慢慢", "故事", "看着", "一点", "笑容", "音乐", "温柔", "画面", "不想", "可爱", "天空", "声音", "身边", "远方", "只能", "忘记", "思念","沉默", "女人", "幸福", "也许", "快乐", "感动", "难过", "跟着", "月光", "一场", "阳光", "表情", "打开", "Music", "安静", "一路", "明白", "放弃", "温暖","心碎", "眼睛", "弦乐", "味道", "从前", "我要", "习惯", "回到", "一遍", "简单", "伤心", "灵魂", "自由", "一口", "生命", "甜蜜", "拥有", "风景", "距离","生活", "美"]

value=[88, 80, 72, 69, 66, 61, 60, 59, 57, 56, 55, 55, 54, 50, 50, 48, 47, 46, 45, 42, 41, 40, 38, 38, 37, 37, 37, 36, 36,36, 36, 35, 34, 34, 34, 32, 32, 32, 31, 31, 30, 30, 29, 29, 29, 28, 28, 28, 28, 28, 28, 26, 26, 26, 25, 25, 25, 25,25, 24, 24, 24, 24, 24, 24, 24, 24, 23, 23, 23, 23, 23, 22, 22, 22, 22, 22, 22]

wordcloud= WordCloud(width=1300, height=620)

wordcloud.add("", name, value,shape='diamond', word_size_range=[20, 100])

wordcloud.render(r"D:\学习\untitled4\allpicture\歌词词云.html")

好了！！其实分析语法没那么重要，实验的方法也很多！！

我们来看下可视化的结果吧

其实爬取了很多内容，但是利用的不是很多，以后再来详解吧

在进行数据可视化的过程中我真的发现pyecharts是一个非常nice的工具！！！！collection

版权申明：本站文章部分自网络，如有侵权，请联系：west999com@outlook.com

特别注意：本站所有转载文章言论不代表本站观点，本站所提供的摄影照片，插画，设计作品，如需使用，请与原作者联系，版权归原作者所有