import requests
#1.指定url
url = 'https://www.sogou.com'
#2.发起请求:get返回值为响应对象
response = requests.get(url=url,proxies={'https':'80.78.72.152:39172'})
#3.获取页面数据:text属性返回的是字符串类型的页面数据,content返回的是二进制的页面数据
#page_text = response.content
page_text = response.text
#4
with open('./sogou.html','w',encoding='utf-8') as fp:fp.write(page_text)

#UA反爬机制：对UA进行伪装操作
#请求头信息
import requests
url = 'https://www.qiushibaike.com/'
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36'
}
#headers参数：请求的请求头信息
response = requests.get(url=url,headers=headers)
response

import requests
kdj_info = []
url = "http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=keyword"
city = input("请输入一个城市的名字:>>>")
data = {"cname":"","pid":"","keyword":city,"pageIndex": 1,"pageSize": 10,
}
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}
response = requests.post(url=url, headers=headers, data=data)
result = response.json()# 构建自己想要的数据结构
for index, one in enumerate(result["Table1"]):
#     print(index, one)kdj_info.append({"序号":index + 1, "店名":one["storeName"], "地址":one["addressDetail"]})
for store_info in kdj_info:print(store_info)

View Code

#http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsList
import requests
first_url = "http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsList"
id_list = []
detail_list = []
result_list = []
start_page = int(input("请输入开始页"))
end_page = int(input("请输入结束页"))
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}
for i in range(start_page, end_page+1):data = {'on': 'true','page':str(i),'pageSize': '15','productName':'','conditionType': '1','applyname':'','applysn':''}response = requests.post(url=first_url, data=data, headers=headers).json()for info in response["list"]:id_list.append(info["ID"])
res_url = "http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsById"for id in id_list:data = {"id":id}res_response = requests.post(url=res_url, data=data, headers=headers).json()detail_list.append(res_response)
# print(detail_list)
# 数据结构的构建
for detail in detail_list:result_list.append({"创始人":detail["businessPerson"], "地址":detail["epsAddress"]})for every_info in result_list:print(every_info)

View Code

import requests
from urllib import request,parse
url = 'http://pics.sc.chinaz.com/files/pic/pic9/201812/zzpic15565.jpg'img_data = requests.get(url=url).contentwith open('./siwa.jpg','wb') as fp:fp.write(img_data)
爬取图片的第二种方法
url = 'http://pics.sc.chinaz.com/files/pic/pic9/201812/zzpic15565.jpg'
request.urlretrieve(url=url,filename='./meinv.png')

request.urlopen(url=url).read()

# url = 'https://www.baidu.com/s?wd=人民币'# request.urlopen(url=url)parse.quote('人民币')

import re
a = '''asdfsafhellopass:234455worldafdsf'''
b = re.findall('hello(.*?)world',a)
c = re.findall('hello(.*?)world',a,re.S)
print 'b is ' , b
print 'c is ' , c
运行结果
b is  []
c is  ['pass:\n\t234455\n\t']

#1.检查页面数据是否为动态加载出来的
#2.获取页面源码数据
import re
import requests
from urllib import request
import os
if not os.path.exists('qiutu'):os.mkdir('qiutu')headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}
url = 'https://www.qiushibaike.com/pic/'
page_text = requests.get(url=url,headers=headers).text
#3.解析img标签的src属性值
ex = '<div class="thumb">.*?<img src="(.*?)" alt=.*?</div>'
img_url_list = re.findall(ex,page_text,re.S)
for img_url in img_url_list:img_url = 'https:'+img_urlimgPath = 'qiutu/'+img_url.split('/')[-1]#4.对图片url发请求#5.持久化存储request.urlretrieve(url=img_url,filename=imgPath)print(imgPath+'下载成功!!!')