简介：提取网页保存到txt文件中 + 解析txt文件内容，取出内容

from urllib import request
import re.jsonurl="http://www.163.com"
response = request.urlopen(url).read().decode("gbk")
# 写入一个txt文件
with open("163/163.txt","w",encoding="gbk")as f:f.write(str(response))
# 读取,用json序列化后遍历出来，取出键里面的值
with open("163/163.txt",'r+',encoding="utf-8")as f:content=f.read()
c1=content.replace(" ",'').replace("\n",'') #去掉空格、换行
c2="["+c1+"]"
cc=json.loads.(c2)  #序列化
for i in cc:print(x['title'])   #取出每个项里面“title”键的值print(x['docurl'])

简介：提取美团链接保存在一个txt文件中

#coding=utf-8
from urllib import request
from bs4 import BeautifulSoup
req=request.urlopen("http://hotel.meituan.com/xian/")
content=req.read().decode("utf8")
bsObj=BeautifulSoup(content,"html.parser")
pcontent=bsObj.findAll("a",{"class":"poi-title"})i=1
with open("meituan/url.txt","a+",encoding="utf8") as f:for x in pcontent:f.write(x['href']+"\n")    #取出 标签 里面的信息f.write(x.get_text())    #取出正文print("第"+'int(%s)'%(i)+"条url")i+=1

范例：功能实现后提高代码质量

#coding=utf-8
#获取当前地址下的所有酒店url地址
from urllib import request,error
from bs4 import BeautifulSoup
import json
for page in range(3):url="https://ihotel.meituan.com/hbsearch/HotelSearch?utm_medium=pc&version_name=999.9&cateId=20&attr_28=129&uuid=12B729E22135402D5CBC1432A179A735CF81DF50626153919EC2C66D46DCB233%401517811001478&cityId=42&offset="+str(page*20)+"&limit=20&startDay=20180205&endDay=20180205&q=&sort=defaults"try:req=request.urlopen(url)content=req.read().decode("utf8")
##                bsObj=BeautifulSoup(content,"html.parser")
##                pcontent=bsObj.findAll("a",{"class":"poi-title"})content_dict=json.loads(content)with open("meituan/url.txt","a+",encoding="utf8") as f:for x in content_dict['data']['searchresult']:print(x['poiid'])hotel_url="http://hotel.meituan.com/%s/"%x['poiid']f.write(hotel_url+"\n")except error.URLError as e:print(e.reason)

urllib添加代理IP

# -*- coding: UTF-8 -*-
from urllib import requestif __name__ == "__main__":#访问网址url = 'http://2017.ip138.com/ic.asp'#url = 'http://www.whatismyip.com.tw'#这是代理IPproxy = {'http':'113.124.226.174:808'}#创建ProxyHandlerproxy_support = request.ProxyHandler(proxy)#创建Openeropener = request.build_opener(proxy_support)#添加UserAngentopener.addheaders = [('User-Agent','Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36'),('Host','www.whatismyip.com.tw')    #这个网站能检测IP地址，所以用这个作为示例]#安装OPenerrequest.install_opener(opener)#使用自己安装好的Openerresponse = request.urlopen(url)#读取相应信息并解码html = response.read().decode("gbk")#打印信息print(html)