header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36','Host': 'tuan.bookschina.com','Referer': 'http://tuan.bookschina.com/','Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8','Accept-Encoding': 'gzip, deflate','Accept-Language': 'zh-CN,zh;q=0.9'
}

url = 'http://tuan.bookschina.com/'
response = requests.get(url, headers = header) #模仿浏览器登录
response.encoding = 'utf-8'
soup = BeautifulSoup(response.text,'html.parser')
for item in soup.select('div .taoListInner ul li'):print(item.select('h2')[0].text) #返回对象为数组print(item.select('.salePrice')[0].text)print(item.select('img')[0].get('src')) #get方法用来取得tab内部的属性值

url = 'http://tuan.bookschina.com/Home/GroupList?Type=0&Category=0&Price=0&Order=11&Page=2&Tyjson=true'
response = requests.get(url)
result = json.loads(response.text)
bookinfo = {}
for data in result['Data']:bookinfo['bookName'] = data['book_name']bookinfo['price'] = data['group_price']bookinfo['iconLink'] = data['group_image']
print(url)

import requests
from bs4 import BeautifulSoup
import json
import csvdef parse_one_page():header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36','Host': 'tuan.bookschina.com','Referer': 'http://tuan.bookschina.com/','Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8','Accept-Encoding': 'gzip, deflate','Accept-Language': 'zh-CN,zh;q=0.9'}url = 'http://tuan.bookschina.com/'response = requests.get(url, headers = header) #模仿浏览器登录response.encoding = 'utf-8'soup = BeautifulSoup(response.text,'html.parser')for item in soup.select('div .taoListInner ul li'):print(item.select('h2')[0].text) #返回对象为数组print(item.select('.salePrice')[0].text)print(item.select('img')[0].get('src')) #get方法用来取得tab内部的属性值def dynamtic_scraping_data(page, headers, fileName):for i in range(page):url = 'http://tuan.bookschina.com/Home/GroupList?Type=0&Category=0&Price=0&Order=11&Page=' + str(i) + '&Tyjson=true'response = requests.get(url)result = json.loads(response.text)bookinfo = {}for data in result['Data']:bookinfo['bookName'] = data['book_name']bookinfo['price'] = data['group_price']bookinfo['iconLink'] = data['group_image']write_csv_rows(fileName,headers,bookinfo)print(url)def write_csv_headers(path, headers):with open(path, 'a', encoding='gb18030', newline='') as f:f_csv = csv.DictWriter(f, headers)f_csv.writeheader()def write_csv_rows(path, headers, rows):with open(path, 'a', encoding='gb18030', newline='') as f:f_csv = csv.DictWriter(f, headers)# 如果写入数据为字典，则写入一行，否则写入多行if type(rows) == type({}):f_csv.writerow(rows)else:f_csv.writerows(rows)
def main(page):# parse_one_page() #Tip: beautifulSoup testcsv_filename = "bookInfo.csv"headers = ['bookName', 'price', 'iconLink']write_csv_headers(csv_filename,headers)dynamtic_scraping_data(page, headers, csv_filename)if __name__ == '__main__':main(20) #input page num to start