08：Python数据分析之pandas学习

1.1 数据结构介绍

　　参考博客：http://www.cnblogs.com/nxld/p/6058591.html

　　1、pandas介绍

　　　　　　1. 在pandas中有两类非常重要的数据结构，即序列Series和数据框DataFrame。
　　　　　　2. Series类似于numpy中的一维数组，除了通吃一维数组可用的函数或方法，而且其可通过索引标签的方式获取数据，还具有索引的自动对齐功能；
　　　　　　3. DataFrame类似于numpy中的二维数组，同样可以通用numpy数组的函数和方法，而且还具有其他灵活应用，后续会介绍到。

　　2、Series创建的三种方式

　　　　1、通过一维数组创建序列

import numpy as np, pandas as pd
arr1 = np.arange(10)
print arr1,type(arr1)   # [0 1 2 3 4 5 6 7 8 9] <type 'numpy.ndarray'>

s1 = pd.Series(arr1)
print s1,type(s1)
# 0    0
# 1    1
# 2    2
# 3    3
# 4    4
# 5    5
# 6    6
# 7    7
# 8    8
# 9    9
# dtype: int64 <class 'pandas.core.series.Series'>

通过一维数组创建序列

　　　　2、通过字典的方式创建序列

import numpy as np, pandas as pd
dic1 = {'a':10,'b':20,'c':30,'d':40,'e':50}
s2 = pd.Series(dic1)
print s2, type(s2)
# a    10
# b    20
# c    30
# d    40
# e    50
# dtype: int64 <class 'pandas.core.series.Series'>

通过字典的方式创建序列

　　　　3、通过DataFrame中的某一行或某一列创建序列

　　3、DataFrame创建的三种方式

　　　　1、通过二维数组创建数据框

import numpy as np, pandas as pd
arr2 = np.array(np.arange(12)).reshape(4,3)
print arr2,type(arr2)
# [[ 0  1  2]
#  [ 3  4  5]
#  [ 6  7  8]
#  [ 9 10 11]]

df1 = pd.DataFrame(arr2)
print df1,type(df1)
#    0   1   2
# 0  0   1   2
# 1  3   4   5
# 2  6   7   8
# 3  9  10  11

通过二维数组创建数据框

　　　　2.1 通过字典的方式创建数据框

import numpy as np, pandas as pd
dic2 = {'a':[1,2,3,4],'b':[5,6,7,8],'c':[9,10,11,12],'d':[13,14,15,16]}
df2 = pd.DataFrame(dic2)
print df2#    a  b   c   d
# 0  1  5   9  13
# 1  2  6  10  14
# 2  3  7  11  15
# 3  4  8  12  16

法1：字典列表生成DataFrame

import numpy as np, pandas as pd
dic3 = {'one':{'a':1,'b':2,'c':3,'d':4},'two':{'a':5,'b':6,'c':7,'d':8},'three':{'a':9,'b':10,'c':11,'d':12}}
df3 = pd.DataFrame(dic3)
print df3, type(df3)#    one  three  two
# a    1      9    5
# b    2     10    6
# c    3     11    7
# d    4     12    8

法2：嵌套字典生成DataFram

# -*- coding: utf-8 -*-
import json
import pandas as pdd = {"slagroupcount": [{"g_sla": 99.943755250038564,"weight": 20.0,"g_t_v": 19.988751050007714,"sla_nums": 14,"id": 1,"name": "大数据"},{"g_sla": 99.994763756058816,"weight": 20.0,"g_t_v": 19.998952751211764,"sla_nums": 6,"id": 2,"name": "基础架构"},],"slacount": 99.611111411465515
}result = {}
gcounts = []
subs = []for i in range(10):day_of_result = dgcounts.append(float(day_of_result['slacount']))  # "slacount": 99.611111411465515subs += day_of_result['slagroupcount']  # slagroupcount是一个列表，列表中包含多个字典
result['slacount'] = sum(gcounts) / len(gcounts)
print subs
df = pd.DataFrame(subs)  # subs = [{},{},,{},{}....]
# print df

g = df.groupby('name').mean()  # 将数据按照name分组计算平均值
print g
'''  # 下面是g的打印结果（按照name分组，求出各项平均值）g_sla      g_t_v      id     sla_nums  weight
name
基础架构  99.994764   19.998953   2         6     20.0
大数据    99.943755   19.988751   1        14     20.0
'''

举例：df.groupby对数据框进行分组

# -*- coding: utf-8 -*-
import json
import pandas as pd
'''一：这里字典d是GroupCountResult表中result字段中的一条数据'''
d = {"slagroupcount": [{"g_sla": 99.943755250038564,"weight": 20.0,"g_t_v": 19.988751050007714,"sla_nums": 14,"id": 1,"name": "大数据"},{"g_sla": 99.994763756058816,"weight": 20.0,"g_t_v": 19.998952751211764,"sla_nums": 6,"id": 2,"name": "基础架构"},],"slacount": 99.611111411465515
}'''二：模拟获取最近10天sla平均值：下面使用for循环伪造从GroupCountResult表中取出了10条数据，进行平均值计算'''
result = {}
gcounts = []
subs = []
for i in range(10):day_of_result = dgcounts.append(float(day_of_result['slacount']))  # "slacount": 99.611111411465515subs += day_of_result['slagroupcount']  # slagroupcount是一个列表，列表中包含多个字典
df = pd.DataFrame(subs)  # subs = [{},{},,{},{}....]
g = df.groupby('name').mean()  # 将数据按照name分组计算平均值
print g
'''  # 下面是g的打印结果（按照name分组，求出各项平均值）g_sla      g_t_v      id     sla_nums  weight
name
基础架构  99.994764   19.998953   2         6     20.0
大数据    99.943755   19.988751   1        14     20.0
''''''三：将利用pandas计算出来的结果循环到字典中'''
result = {}
result['slagroupcount'] = []
for index, row in g.iterrows():result['slagroupcount'].append({'name': row.name,'id': int(row.id),'weight': row.weight,'sla_nums': row.sla_nums,'g_sla': row.g_sla,'g_t_v': row.g_t_v})
print result['slagroupcount']
'''  # 这里的d就是求出上面10条平均值后生成的字典
d = {"slagroupcount": [{"g_sla": 99.943755250038564,"weight": 20.0,"g_t_v": 19.988751050007714,"sla_nums": 14,"id": 1,"name": "大数据"},{"g_sla": 99.994763756058816,"weight": 20.0,"g_t_v": 19.998952751211764,"sla_nums": 6,"id": 2,"name": "基础架构"},],"slacount": 99.611111411465515
}
'''

举例2：字典生成数据框，分组求平均值，然后将结果存入新字典

　　　　2.2 对数据框分组求值

# -*- coding: utf-8 -*-
import json
import pandas as pdli = [
{'name': 'Hospital01', 'abbreviation': 'sdhospital','domain': '', 'service': 'mongodb', 'sla': 97.07472},
{'name': 'Hospital01', 'abbreviation': 'sdhospital','domain': '', 'service': 'redmine', 'sla': 93.07472},
{'name': 'Hospital01', 'abbreviation': 'sdhospital','domain': '', 'service': 'mongodb', 'sla': 95.07472},
{'name': 'Hospital01', 'abbreviation': 'sdhospital','domain': '', 'service': 'redmine', 'sla': 98.07472},
{'name': 'Hospital02', 'abbreviation': 'sysucc','domain': '', 'service': 'redmine', 'sla': 87.07472},
{'name': 'Hospital02', 'abbreviation': 'sysucc','domain': '', 'service': 'mongodb', 'sla': 73.07472},
{'name': 'Hospital02', 'abbreviation': 'sysucc','domain': '', 'service': 'redmine', 'sla': 55.07472},
{'name': 'Hospital02', 'abbreviation': 'sysucc','domain': '', 'service': 'mongodb', 'sla': 78.07472},
]# 第一步：将列表字典转换成数据框
df = pd.DataFrame(li)  # 将列表字典转换成数据框# 第二步：将数据按照name分组计算平均值
g = df.groupby('name').mean()  # 将数据按照name分组计算平均值
# print g
'''sla
name
Hospital01  95.82472
Hospital02  73.32472
'''# 第三步：将二中分组后的值转换成字典
print g.to_dict()
'''
{"sla": {"Hospital01": 95.82472, "Hospital02": 73.32472}
}
'''

例1：对其中一个指标进行分组求值

# -*- coding: utf-8 -*-
import json
import pandas as pdli = [
{'name': 'Hospital01', 'abbreviation': 'sdhospital','domain': '', 'service': 'mongodb', 'sla': 97.07472},
{'name': 'Hospital01', 'abbreviation': 'sdhospital','domain': '', 'service': 'redmine', 'sla': 93.07472},
{'name': 'Hospital01', 'abbreviation': 'sdhospital','domain': '', 'service': 'mongodb', 'sla': 95.07472},
{'name': 'Hospital01', 'abbreviation': 'sdhospital','domain': '', 'service': 'redmine', 'sla': 98.07472},
{'name': 'Hospital02', 'abbreviation': 'sysucc','domain': '', 'service': 'redmine', 'sla': 87.07472},
{'name': 'Hospital02', 'abbreviation': 'sysucc','domain': '', 'service': 'mongodb', 'sla': 73.07472},
{'name': 'Hospital02', 'abbreviation': 'sysucc','domain': '', 'service': 'redmine', 'sla': 55.07472},
{'name': 'Hospital02', 'abbreviation': 'sysucc','domain': '', 'service': 'mongodb', 'sla': 78.07472},
]# 第一步：将列表字典转换成数据框
df = pd.DataFrame(li)  # 将列表字典转换成数据框# 第二步：将数据框按照 service，name，abbreviation 同时分组
service_name_group = df.groupby([df['service'], df['name'], df['abbreviation']]).mean()
# print service_name_group
'''
service name       abbreviation
mongodb Hospital01 sdhospital    96.07472Hospital02 sysucc        75.57472
redmine Hospital01 sdhospital    95.57472Hospital02 sysucc        71.07472
'''# 第三步：将分组后的结果转换成字典
# print service_name_group.to_dict()
'''
{'sla': {('redmine', 'Hospital01', 'sdhospital'): 95.57472,('redmine', 'Hospital02', 'sysucc'): 71.07472,('mongodb', 'Hospital02', 'sysucc'): 75.57472,('mongodb', 'Hospital01', 'sdhospital'): 96.07472}
}
'''# 第四步：将转换成的字典转换成我们想要的字典格式
context = {}
for k, v in service_name_group.to_dict()['sla'].items():context.setdefault(k[0], [])  # {'mongodb': [], 'redmine': []}context[k[0]].append({'name': k[1], 'sla': v, 'abbreviation': k[2]})
''' 这是for循环k，v的结果
('redmine', 'Hospital01', 'sdhospital') 95.57472
('redmine', 'Hospital02', 'sysucc') 71.07472
('mongodb', 'Hospital02', 'sysucc') 75.57472
('mongodb', 'Hospital01', 'sdhospital') 96.07472
'''
# print context
# 这里d是我们最终想要得到的结果
d = {"mongodb": [{"abbreviation": "sysucc","name": "Hospital02","sla": 75.57472},{"abbreviation": "sdhospital","name": "Hospital01","sla": 96.07472}],"redmine": [{"abbreviation": "sdhospital","name": "Hospital01","sla": 95.57472},{"abbreviation": "sysucc","name": "Hospital02","sla": 71.07472}]
}

例2：同时对多个指标进行分组求值

# -*- coding: utf-8 -*-
import json
import pandas as pdli = [{'name':'zhangsan','times':'first','math':88,'chinese':82},{'name':'zhangsan','times':'second','math':84,'chinese':83},{'name':'zhangsan','times':'third','math':85,'chinese':87},{'name': 'lisi', 'times': 'first', 'math': 88, 'chinese': 82},{'name': 'lisi', 'times': 'second', 'math': 84, 'chinese': 83},{'name': 'lisi', 'times': 'third', 'math': 85, 'chinese': 87},
]# 第一步：将列表字典转换成数据框
df = pd.DataFrame(li)  # subs = [{},{},,{},{}....]# 第二步：将数据框按照name分组
g = df.groupby([df['name']]).mean()
# print g
'''chinese       math
name
lisi         84.0  85.666667
zhangsan     84.0  85.666667
'''# 第三步：将利用pandas计算出来的结果循环到字典中
result = []
for index, row in g.iterrows():result.append({'name': row.name,'math': int(row.math),'chinese': row.chinese,})
# print result
ret_li = [{"chinese": 84,"name": "lisi","math": 85},{"chinese": 84,"name": "zhangsan","math": 85}
]

例3：对一个指标多个数据分组求值(求zhangsan，lisi两个用户三次考试语文数学平均成绩)

　　　　2.3 对数据框进行过滤查询

# -*- coding: utf-8 -*-
import json
import pandas as pdli = [{'name':'zhangsan','times':'first','math':88,'chinese':82},{'name':'zhangsan','times':'second','math':84,'chinese':83},{'name':'zhangsan','times':'third','math':85,'chinese':87},{'name': 'lisi', 'times': 'first', 'math': 88, 'chinese': 82},{'name': 'lisi', 'times': 'second', 'math': 84, 'chinese': 83},{'name': 'lisi', 'times': 'third', 'math': 85, 'chinese': 87},
]# 第一步：将列表字典转换成数据框
df = pd.DataFrame(li)  # subs = [{},{},,{},{}....]# 第二步：过滤出zhangsan用户，第一次考试的结果
result = df[(df['name'] == 'zhangsan') & (df['times']=='first')]
# result = df[(df['name'] == 'zhangsan') | (df['times']=='first')]  # 过滤出name='zhangsan' 或者 times='first' 的条目# 第三步：将第二步中过滤的结果添加到字典中
li = []
for index, row in result.iterrows():li.append({'姓名':row['name'],'第几次考试':row['times'],'数学成绩':row['math'],'语文成绩':row['chinese']})
print json.dumps(li)'''
[{"第几次考试": "first","语文成绩": 82,"数学成绩": 88,"姓名": "zhangsan"
}]
'''

例1：对数据框进行条件过滤

1.2 数据索引index

　　1、通过索引值或索引标签获取数据

import numpy as np, pandas as pd#1、通过列表生成Series
s4 = pd.Series(np.array([1,2,3,4]))
print s4
# 0    1
# 1    2
# 2    3
# 3    4#2、为Series自定义的索引值
s4.index = ['a','b','c','d']
print s4
# a    1
# b    2
# c    3
# d    4#3、通过两种索引均可获取到值
print s4[3],s4['d']  # 4 4

通过索引值或索引标签获取数据

　　2、自动化对齐

#-*- coding:utf8 -*-
import numpy as np, pandas as pds5 = pd.Series(np.array([10,15,20,30]), index = ['a','b','c','d'])
s6 = pd.Series(np.array([12,11,13,15]), index = ['a','c','g','b'])
print s5 + s6
# a    22.0
# b    30.0
# c    31.0
# d     NaN
# g     NaN# 说明：由于s5中的d和s6中的g没有对应的所有，所以数据的运算会产生两个缺失值NaN
# 注意，这里的算术结果就实现了两个序列索引的自动对齐，而非简单的将两个序列加总或相除。
# 对于数据框的对齐，不仅仅是行索引的自动对齐，同时也会自动对齐列索引（变量名）

自动化对齐

1.3 统计分析

#-*- coding:utf8 -*-
import numpy as np, pandas as pdnp.random.seed(1234)
d1 = pd.Series(2*np.random.normal(size = 100)+3)   # 生成Series 100个

d1.count() #非空元素计算
d1.min() #最小值
d1.max() #最大值
d1.idxmin() #最小值的位置，类似于R中的which.min函数
d1.idxmax() #最大值的位置，类似于R中的which.max函数
d1.quantile(0.1) #10%分位数
d1.sum() #求和
d1.mean() #均值
d1.median() #中位数
d1.mode() #众数
d1.var() #方差
d1.std() #标准差
d1.mad() #平均绝对偏差
d1.skew() #偏度
d1.kurt() #峰度
d1.describe() #一次性输出多个描述性统计指标

统计分析基本使用

转载于:https://www.cnblogs.com/xiaonq/p/9036911.html