import requests
import re
import urllibdef requets(flow):flow.request.headers['User-Agent'] = 'MitmProxy'print(flow.request.headers)

from mitmproxy import ctx
import jsondef response(flow):print('获取列表数据中...')#url = 'http://mapi.dangdang.com/index.php?cat_path_text=%E6%80%BB%E6%A6%9C&img_size=b&bang_name_text=%E5%9B%BE%E4%B9%A6%E7%95%85%E9%94%80%E6%A6%9C&access-token=&permanent_id=20190405192112099238317917598184509&ischildren=0&bang_name=bestsell&user_client=android&pagesize=10&union_id=537-100380&time_code=849e796eaa9e66cae342dd1f47c5f032&action=bang_tushu&page=5&global_province_id=151&cat_path=01.00.00.00.00.00&client_version=9.4.2&udid=c906965ad731be703305409f738a1bad&timestamp=1556893369'url = flow.request.urlif flow.request.url.startswith(url):text = flow.response.text#print(text)data = json.loads(text)books = data.get('products')for book in books:info = {'书名': book.get('product_name'),'作者': book.get('author'),'价格': book.get('price').get('dangdang_price'),'封面图片': book.get('img_url'),}ctx.log.info(str(book))with open('D:\\books.txt', 'a', encoding='utf-8') as f:f.write(json.dumps(info, ensure_ascii=False) + '\n')

import pymongo
from pymongo import MongoClient
import jsonclient = pymongo.MongoClient('mongodb://admin:admin@localhost:27017/')
db = client['books']
collection = db['book']with open('D:\\books.txt', 'r+', encoding='utf-8') as f:for i in f.readlines():new = json.loads(i)try:if collection.insert(new):print("成功保存到MongoDB")except Exception:print('someing wrong with MongDB')