1 # Author:K
 2 # 需求：爬取搜狗首页的页面数据
 3
 4 import requests
 5
 6 # step 1:指定url
 7 url = 'https://www.sogou.com/'
 8
 9 # step 2:发起请求
10 response = requests.get(url = url) # get()方法会返回一个相应对象
11
12 # step 3:获取响应数据：
13 page_text = response.text # text方法返回的是字符串形式的相应数据
14 print(page_text)
15
16 # step 4:持久化存储
17 with open('./sogou.html','w',encoding='utf-8') as fp:
18     fp.write(page_text)
19
20 print("爬取数据结束！")

 1 # Author:K
 2 import requests
 3
 4 # UA伪装：
 5 headers = {
 6     "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36"
 7 }
 8
 9 keyword = input("Please input keyword:")
10 url = 'https://www.sogou.com/web?'
11
12 # 参数设置：
13 params = {
14     'query':keyword
15 }
16
17 response = requests.get(url = url,params = params,headers = headers)
18
19 data = response.text
20
21 fileName = keyword + '.html'
22 with open(fileName,'w',encoding='utf-8') as fp:
23     fp.write(data)
24
25 print("爬取结束！")

 1 # Author:K
 2
 3 import requests
 4 import json
 5
 6 url = 'https://fanyi.baidu.com/sug'
 7
 8 headers = {
 9     "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36"
10 }
11
12 kw = input("Please input keyword:")
13 data = {
14     'kw':kw
15 }
16
17 response = requests.post(url = url,data = data,headers = headers)
18
19 dic_obj = response.json() # 要确定服务器响应的son数据才能使用此方法
20
21 # 存储到本地
22 fileName = kw + '.json'
23 fp = open(fileName,'w',encoding='utf-8')
24 json.dump(dic_obj,fp = fp,ensure_ascii=False) # json文件中有中文，不能使用ascii编码
25
26 print("爬取结束！")

 1 # Author:K
 2 import requests
 3 import json
 4
 5 url = 'https://movie.douban.com/j/chart/top_list'
 6
 7 start_movie = input("Please input number that which movie you want to choose first:")
 8 limit_movie = input("Please input number that how many movie one time display:")
 9
10 params = {
11     "type": "5",
12     "interval_id": "100:90",
13     "action": "",
14     "start": start_movie,  # 从第几部电影开始取
15     "limit": limit_movie, # 一次取多少部
16 }
17
18 headers = {
19     "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36"
20 }
21
22 response = requests.get(url = url,params = params,headers = headers)
23
24 list_obj = response.json()
25
26 fileName = "doubanMovie.json"
27 fp = open(fileName,'w',encoding = 'utf-8')
28 json.dump(list_obj,fp = fp,ensure_ascii = False)
29
30 print("Successful crawling!")

 1 # Author:K
 2  #--------------------------为什么查询不出来？？？？
 3 import requests
 4
 5 url = 'http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=keyword'
 6
 7 headers = {
 8     "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36"
 9 }
10
11 #kw = input("Please input keyword:")
12 page_index = input("Which page you want to choose first:")
13 page_size = input("The pageSize is:")
14
15 data = {
16     "cname": " ",
17     "pid": " ",
18     "keyword": "北京",
19     "pageIndex": page_index,
20     "pageSize": page_size,
21 }
22
23 response = requests.post(url = url,data = data,headers = headers)
24 page_text = response.json()
25 print(page_text)
26 fileName = "KFC restaurant.txt"
27 with open(fileName,'w',encoding = 'utf-8') as fp:
28     fp.write(str(page_text))
29
30 print("Successful crawling!")

 1 # Author:K
 2 import requests
 3 import json
 4
 5 url = 'http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsList'
 6
 7 headers = {
 8     "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36"
 9 }
10
11 all_data_list = [] # 存储企业详情数据
12 id_list = []  # 存储企业对应的id
13
14 for page in range(1,5): # !!!!!!!!!!!!为什么这里范围大了就不行？？？？？？？？？
15     data = {
16         "on": " true",
17         "page": str(page),
18         "pageSize": " 15",
19         "productName": " ",
20         "conditionType": " 1",
21         "applyname": " ",
22         "applysn": " ",
23     }
24
25     dic_info = requests.post(url = url,data = data,headers = headers).json()
26
27     for dic in dic_info["list"]:
28         id_list.append(dic["ID"])
29
30 info_url = 'http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsById'
31 for id in id_list:
32     id_data = {
33         "id":id
34     }
35
36     info_response = requests.post(url = info_url,data = id_data,headers = headers)
37     dic_data = info_response.json()
38     all_data_list.append(dic_data)
39
40 # 持久化存储
41 fileName_data = "data.json"
42 fp = open(fileName_data,'w',encoding = 'utf-8')
43 json.dump(all_data_list,fp = fp,ensure_ascii = False)
44
45 print(len(all_data_list))
46 print("successful crawling!")