# 对网页标题的操作 （删除 快车-律师助手 & 免费发布法律咨询 & 咨询发布成功 & 法律快搜）
# 读取数据库数据（基于操作1之后）
engine = create_engine('mysql+pymysql://root:@127.0.0.1:3306/jing?charset=utf8')
sql = pd.read_sql('cleaned_one', engine, chunksize = 10000)# 对网址的操作 （只要.html结尾的&只要包含主网址（lawtime)的&网址中间没有midques_的
for i in sql:d = i[['realIP','fullURL','pageTitle','userID','timestamp_format']]# 只要网址列d['pageTitle'].fillna(u'空',inplace=True)d = d[(d['pageTitle'].str.contains(u'快车-律师助手') == False) & (d['pageTitle'].str.contains(u'咨询发布成功') == False) & \(d['pageTitle'].str.contains(u'免费发布法律咨询') == False) & (d['pageTitle'].str.contains(u'法律快搜') == False)\].copy()# 保存到数据库中d.to_sql('cleaned_two', engine, index = False, if_exists = 'append')

注意：最后发现，对于网页标题需要进行的删除的记录的网址中，均没有.html，因此，操作2可以不必做，操作1已完成工作

1.2.3 第三步，完成删除规则5

def dropduplicate(i): j = i[['realIP','fullURL','pageTitle','userID','timestamp_format']].copy()return jcount6 = [dropduplicate(i) for i in sql]
count6 = pd.concat(count6)
print len(count6) # 2012895
count7 = count6.drop_duplicates(['fullURL','userID','timestamp_format']) # 一定要进行二次删除重复，因为不同的块中会有重复值
print len(count7) #　647300
savetosql(count7, 'cleaned_three')

1.3 查看进行删除操作后的表中的总记录数

# 查看all_gzdata表中的记录数
engine = create_engine('mysql+pymysql://root:@127.0.0.1:3306/jing?charset=utf8')
sql = pd.read_sql('all_gzdata', engine, chunksize = 10000)
lens = 0
for i in sql:temp = len(i)lens = temp + lens
print lens # 837450# 查看cleaned_one表中的记录数
engine = create_engine('mysql+pymysql://root:@127.0.0.1:3306/jing?charset=utf8')
sql1 = pd.read_sql('cleaned_one', engine, chunksize = 10000)
lens1 = 0
for i in sql1:temp = len(i)lens1 = temp + lens1
print lens1 # 1341930# 查看cleaned_two表中的记录数
engine = create_engine('mysql+pymysql://root:@127.0.0.1:3306/jing?charset=utf8')
sql2 = pd.read_sql('cleaned_two', engine, chunksize = 10000)
lens2 = 0
for i in sql2:temp = len(i)lens2 = temp + lens2
print lens2 #2012895# 查看cleaned_three表中的记录数
engine = create_engine('mysql+pymysql://root:@127.0.0.1:3306/jing?charset=utf8')
sql3 = pd.read_sql('cleaned_three', engine, chunksize = 10000)
lens3 = 0
for i in sql3:temp = len(i)lens3 = temp + lens3
print lens3 #1294600

2 数据变换

2.1 识别翻页的网址

识别后删除重复（用户ID和处理后的网址相同）的记录

# 读取数据库数据
engine = create_engine('mysql+pymysql://root:@127.0.0.1:3306/jing?charset=utf8')
sql = pd.read_sql('cleaned_three', engine, chunksize = 10000)l0 = 0
l1 = 0
l2 = 0
for i in sql:d = i.copy()# 获取所有记录的个数temp0 = len(d)l0 = l0 + temp0# 获取类似于http://www.lawtime.cn***/2007020619634_2.html格式的记录的个数# 匹配1 易知，匹配1一定包含匹配2x1 = d[d['fullURL'].str.contains('_\d{0,2}.html')]temp1 = len(x1)l1 = l1 + temp1    # 匹配2# 获取类似于http://www.lawtime.cn***/29_1_p3.html格式的记录的个数x2 = d[d['fullURL'].str.contains('_\d{0,2}_\w{0,2}.html')]temp2 = len(x2)l2 = l2 + temp2x1.to_sql('l1', engine, index=False, if_exists = 'append') # 保存x2.to_sql('l2', engine, index=False, if_exists = 'append') # 保存print l0,l1,l2 #1941900 166365 27780

# 注意：在内部循环中，容易删除不完整，所以需要进行全部读取二次筛选删除
# 【初步筛选】
# 读取数据库数据
engine = create_engine('mysql+pymysql://root:@127.0.0.1:3306/jing?charset=utf8')
sql = pd.read_sql('cleaned_three', engine, chunksize = 10000)
l4 = 0
for i in sql:d = i.copy()# 注意！！！替换1和替换2的顺序不能颠倒，否则删除不完整# 替换1 将类似于http://www.lawtime.cn***/29_1_p3.html下划线后面部分"_1_p3"去掉，规范为标准网址 d['fullURL'] = d['fullURL'].str.replace('_\d{0,2}_\w{0,2}.html','.html')#这部分网址有　9260　个# 替换2 将类似于http://www.lawtime.cn***/2007020619634_2.html下划线后面部分"_2"去掉，规范为标准网址d['fullURL'] = d['fullURL'].str.replace('_\d{0,2}.html','.html') #这部分网址有　55455-9260 = 46195 个d = d.drop_duplicates(['fullURL','userID']) # 删除重复记录(删除有相同网址和相同用户ID的)【不完整】因为不同的数据块中依然有重复数据temp = len(d)l4 = l4 + tempd.to_sql('changed_1', engine, index=False, if_exists = 'append') # 保存print l4 # 1643197

# 【二次筛选】
# 删除重复记录
# 读取数据库数据（基于操作2之后）
engine = create_engine('mysql+pymysql://root:@127.0.0.1:3306/jing?charset=utf8')
sql = pd.read_sql('changed_1', engine, chunksize = 10000)def dropduplicate(i): j = i[['realIP','fullURL','pageTitle','userID','timestamp_format']].copy()return jcounts1 = [dropduplicate(i) for i in sql]
counts1 = pd.concat(counts1)
print len(counts1)# 1095216
a = counts1.drop_duplicates(['fullURL','userID'])
print len(a)# 528166
savetosql(a, 'changed_2')

# 查看经过数据变换替换后的数据是否替换干净
# 读取数据库数据
engine = create_engine('mysql+pymysql://root:@127.0.0.1:3306/jing?charset=utf8')
sql = pd.read_sql('changed_2', engine, chunksize = 10000)l0 = 0
l1 = 0
l2 = 0
for i in sql:d = i.copy()# 获取所有记录的个数temp0 = len(d)l0 = l0 + temp0# 获取类似于http://www.lawtime.cn***/2007020619634_2.html格式的记录的个数# 匹配1 易知，匹配1一定包含匹配2x1 = d[d['fullURL'].str.contains('_\d{0,2}.html')]temp1 = len(x1)l1 = l1 + temp1    # 匹配2# 获取类似于http://www.lawtime.cn***/29_1_p3.html格式的记录的个数x2 = d[d['fullURL'].str.contains('_\d{0,2}_\w{0,2}.html')]temp2 = len(x2)l2 = l2 + temp2print l0,l1,l2# 528166 0 0表示已经删除成功

2.2 网址正确分类

# 读取数据库数据
engine = create_engine('mysql+pymysql://root:@127.0.0.1:3306/jing?charset=utf8')
sql = pd.read_sql('changed_2', engine, chunksize = 10000)def countzhishi(i):j = i[['fullURL']].copy()j['type'] = 'else'j['type'][j['fullURL'].str.contains('(info)|(faguizt)')] = 'zhishi'j['type'][j['fullURL'].str.contains('(ask)|(askzt)')] = 'zixun'return j
counts2 = [countzhishi(i) for i in sql]
counts2 = pd.concat(counts2)
counts2['type'].value_counts()

e.columns = ['fullURL', 'type1', 'type2']
print e.head()f.columns = ['fullURL', 'type1', 'type2']
print f.head()# 将两类处理过二级类别的记录合并，求二级类别的交集
g = pd.concat([e,f])
h = g['type2'].value_counts()# 求两类网址中的二级类别数，由结果可知，两类网址的二级类别的集合的并集满足所需条件
len(e['type2'].value_counts()) # 66
len(f['type2'].value_counts()) # 31
len(g['type2'].value_counts()) # 69print h.head()print h.index # 列出知识类别下的所有的二级类别

2.2.4 第四步 *将二级类别分别存储到数据库中

 detailtypes = h.indexfor i in range(len(detailtypes)):x = g[g['type2'] == h.index[i]]savetosql(x,h.index[i])

2.2.5 第五步 *用正则表达式匹配出网址中三级类别

# 复制e的备份进行处理，避免操作中改变了数据
q = e.copy()
q['type3'] = np.nan
resultype3 = DataFrame([],columns=q.columns)
for i in range(len(h.index)):pattern2 = re.compile('/info/'+h.index[i]+'/(.*?)/',re.S)current = q[q['type2'] == h.index[i]]print current.head()for j in range(len(current)):findresult = re.findall(pattern2, current.iloc[j,0])if findresult == []: # 若匹配结果是空，则将空值进行赋值给三级类别current.iloc[j,3] = np.nanelse:current.iloc[j,3] = findresult[0]resultype3 = pd.concat([resultype3,current])# 将处理后的数据拼接
resultype3.head()

resultype3.set_index('fullURL',inplace=True)
resultype3.head(10)

# 统计婚姻类下面的三级类别的数目
j = resultype3[resultype3['type2'] == 'hunyin']['type3'].value_counts()
print len(j) # 145
j.head()

2.2.6  第六步 *目标：将类别3按照每类降序排列，然后保存

# 方式1
Type3nums = resultype3.pivot_table(index = ['type2','type3'], aggfunc = 'count')
# 方式2: Type3nums = resultype3.groupby([resultype3['type2'],resultype3['type3']]).count()
r = Type3nums.reset_index().sort_values(by=['type2','type1'],ascending=[True,False])
r.set_index(['type2','type3'],inplace = True)
#保存的表名命名格式为“2_2_k此表功能名称”，此表表示生成的第1张表格，功能为Type3nums：得出所有三级类别
r.to_excel('2_2_3Type3nums.xlsx')
r

2.2.7 属性规约

获取后续建模需要的数据

咨询（ask）和婚姻（hunyin）数据

# 将满足需求的存到数据库中去
# 方法一：
# 读取数据库数据
engine = create_engine('mysql+pymysql://root:@127.0.0.1:3306/jing?charset=utf8')
sql = pd.read_sql('changed_2', engine, chunksize = 10000)
l1 = 0
l2 = 0
for i in sql:zixun = i[['userID','fullURL']][i['fullURL'].str.contains('(ask)|(askzt)')].copy()l1 = len(zixun) + l1hunyin = i[['userID','fullURL']][i['fullURL'].str.contains('hunyin')].copy()    l2 = len(hunyin) + l2zixun.to_sql('zixunformodel', engine, index=False,if_exists = 'append')hunyin.to_sql('hunyinformodel', engine, index=False,if_exists = 'append')
print l1,l2 # 393185 16982# 方法二：
m = counts2[counts2['type'] == 'zixun']
n =counts2[counts2['fullURL'].str.contains('hunyin')]
p = m[m['fullURL'].str.contains('hunyin')]
p # 结果为空，可知，包含zixun的页面中不包含hunyin，两者没有交集
savetosql(m,'zixun')
savetosql(n,'hunyin')

备注：本章完整代码请见点击打开链接