1 data.isnull()#查看所有缺失值
2 data.isnull().any()#获取含有缺失值的列
3  data.isnull().all()#获取全部为NA的列

data2=data.dropna()

data.index=data['name']#将第一列作为索引data=data.drop(['name'],axis=1)#删除第一列
nan_model=Imputer(missing_values='NaN',strategy='mean',axis=0)#按照行均值替换对应缺失值。
nan_result=nan_model.fit_transform(data)

data.fillna(0) #缺失值用0替换
data.fillna(method='pad')#用前面的值替换
data.fillna(method='backfill')#用后面的值替换

import pandas as pd
import numpy as np
from sklearn.preprocessing import Imputerdf = pd.DataFrame(np.random.randn(6,4),columns = list('abcd'))
df.iloc[4,3] = np.nan#iloc是选择行和列
df.loc[3] = np.nan#loc是选择行print(df)print(df.loc[0:2,3]) nan_all=df.isnull() print(nan_all) drop = df.dropna()#丢弃包含缺失值的行记录 print(drop)

# 删除完全缺失的行，若不指定参数how，则删除的是所有含有nan的行
>>> data.shape
(6578, 7)
>>> data_drop_nan = data.dropna(how='all')
>>> data_drop_nan.shape
(6577, 7)
>>> data.dropna().shape (6575, 7) # 对部分缺失行进行填充 # 用后一行的值填充前一行的缺失值 >>> data_drop_nan.fillnan(method='backfill') # 对指定列填充指定值 >>> data_drop_na.fillna({'购药时间'：'2018-01-20 星期三', '社保卡号': '1.338953e+07'})

data['price'][data['price']==0]=None
x = 0
for i in data.columns:for j in range(len(data)):if (data[i].isnull()) [j]:data[i][j]='36' x+=1 print(x)

from imblearn.over_sampling import SMOTE
#eg1
X_resampled_smote, y_resampled_smote = SMOTE().fit_sample(X, y)#eg2over_samples = SMOTE(random_state=1234)over_samples_X,over_samples_y = over_samples.fit_sample(X_train, y_train)

from imblearn.under_sampling
import ClusterCentroidscc = ClusterCentroids(random_state=0)
X_resampled, y_resampled = cc.fit_sample(X, y)

1 pandas.DataFrame.duplicated(self, subset=None, keep=’first’) 

pandas.DataFrame.drop_duplicates(self, subset=None, keep=’first’, inplace=False)pandas.Series.drop_duplicates(self, keep=’first’, inplace=False)

(dataframe.colname.str.contains("somestring")).sum

dataframe["newcol"]  =  dataframe.somecol.map{ {"female":0 , "male": 1} )

import numpy as npdata.replace([-999,-1000], np.nan)
data.replace({-999: np.nan, -1000:0})

#异常值处理
da = data.values#重新赋值data
#异常值处理，将commments大于200000的数据comments设置为58
cont_clou = len(da)#获取行数
#遍历数据进行处理
for i in range(0,cont_clou):if(data.values[i][3]>200000):#print(data.values[i][3])da[i][3]='58'#print(da[i][3])

import pandas as pd
data = {'state':['Ohio','Ohio','Merry','Nevaio','Nevada'],'year':[2000,2001,2002,2001,2002],'pop':[1.5,1.7,3.6,2.4,2.9]
}
frame = pd.DataFrame(data,columns=['year','state','pop','debt'],index=['one','two','three','four','five'])

frame.ix[frame['state'].str.contains('io'),['debt']] = 'aa'

frame.ix[frame['state']=='Ohio',['debt']] = 'aa'

frame.ix[(frame['state']=='Ohio')|(frame['state']=='Merry'),['debt']] = 'aa'

#年龄数据
ages = [20, 22, 25, 27, 21, 23, 37, 31, 61, 45, 41, 32]
# 分箱的边界
bins = [18, 25, 35, 60, 100]
cats = pd.cut(ages, bins)
print(type(cats))
# Categorical对象
cats
#获取分箱编码
cats.codes #返回分箱便捷索引 cats.categories #统计箱中元素的个数 pd.value_counts(cats) #带标签的分箱 group_names = ['Youth', 'YoungAdult', 'MiddleAged', 'Senior'] cats = pd.cut(ages, bins, labels = group_names) cats cats.get_values()

 def get_period(row):hour = row['起飞时间'].hourif hour in range(6,12):return '上午'elif hour in range(12,18):return '下午'else:return '夜间'

df = pd.DataFrame({'key': ['b', 'b', 'a', 'c', 'a', 'b'], 'data1': range(6)})
dfpd.get_dummies(df['key'])

from numpy import set_printoptions
from sklearn.preprocessing import Binarizer
array=data.values
X=array[:,0:8]#分离出输入变量
Y=array[:,8]#分离出输出变量
transformer=Binarizer(threshold=0.0).fit(X)
newX=transformer.transform(X)
set_printoptions(precision=3)
print(newX.head(10))

data = {'Dave': 'dave@google.com', 'Steve': 'steve@gmail.com', 'Rob': 'rob@gmail.com', 'Wes': np.nan}
data = pd.Series(data)
data#字符串列元素中是否包含子字符串
data.str.contains('gmail')#字符串列切片操作
data.str[:5]split_df = data.str.split('@', expand=True) split_df split_df = data.str.split('@') split_df split_df[0].str.cat(split_df[1], sep='@')

import pandas as pd
def my_min(a, b):return min(abs(a),abs(b))s = pd.Series([10.0247,10.0470, 10.0647,10.0761,15.0800,10.0761,10.0647,10.0470,10.0247,10.0,9.9753,9.9530,9.9353,9.9239,18.92,9.9239,9.9353,9.9530,9.9753,10.0])
df = pd.DataFrame(s)
df.columns=['value'] df['val_1'] = df['value'].diff() df['val_2'] = df['val_1'].shift(-1) df['val'] = df.apply(lambda row: my_min(row['val_1'], row['val_2']), axis=1)

df['new_col']=list(map(lambda x,y: function(x,y), df['col1'], df['col2']))

from numpy import set_printoptions
from sklearn.preprocessing import MinMaxScaler
array=data.values
X=array[:,0:8]#分离出输入变量
Y=array[:,8]#分离出输出变量
transformer=MinMaxScaler(feature_range=(0,1))#指定转换范围为0到1之间
newX=transformer.fit_transform(X)#数据转换
set_printoptions(precision=3)
print(newX.head(10))

from numpy import set_printoptions
from sklearn.preprocessing import StandardScaler
array=data.values
X=array[:,0:8]#分离出输入变量
Y=array[:,8]#分离出输出变量
transformer=StandardScaler().fit(X)
newX=transformer.transform(X)
set_printoptions(precision=3)
print(newX.head(10))

from numpy import set_printoptions
from sklearn.preprocessing import Normalizer
array=data.values
X=array[:,0:8]#分离出输入变量
Y=array[:,8]#分离出输出变量
transformer=Normalizer().fit(X)
newX=transformer.transform(X)
set_printoptions(precision=3)
print(newX.head(10))