Web of Science爬虫实战（Post方法）

一．概述

本次爬虫主要通过论文的标题来检索出该论文，从而爬取该论文的被引量，近180天下载量以及全部下载量。这里使用的是Web of Scienece 核心合集，并且使用python的requests 库中post方法进行爬取，此外为了加快爬取速度2.0版本采取了多线程的技术

二．网站及爬取策略分析

首先点击http://apps.webofknowledge.com/UA_GeneralSearch_input.do?product=UA&search_mode=GeneralSearch&SID=5BKzcH5qIlQiOPN9Wjv&preferencesSaved=

这个链接进入相应页面然后按下F12点击右侧的network页面，如下图所示：

图1:检索页面

然后在右侧红框内选择web of science 核心合集以及标题按钮，然后输入标题我这里以Blackcarbon in soils from different land use areas of Shanghai, China: Level,sources and relationship with polycyclic aromatic hydrocarbons为例，下图中右侧红框内内容即为需要用post方法提交的数据。

图2.post提交数据来源

点击检索即可进入我们需要爬取的页面如下

图3.爬取页面

看到右侧用红框标出的页面鼠标移到上面右键检查即课打开该开发者页面，具体分析里面的html

三．爬虫代码

下面即是完整的python代码，也可以访问我的github https://github.com/jgzquanquan/Spyder_wos

title_wos_1.0版本

import re
# from threading import Thread
from multiprocessing import Process
from multiprocessing import Manager
import requests
import time
import xlrd
from bs4 import BeautifulSoup
from lxml import etreeclass SpiderMain(object):def __init__(self, sid, kanming):self.hearders = {'Origin': 'https://apps.webofknowledge.com','Referer': 'https://apps.webofknowledge.com/UA_GeneralSearch_input.do?product=UA&search_mode=GeneralSearch&SID=R1ZsJrXOFAcTqsL6uqh&preferencesSaved=','User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36",'Content-Type': 'application/x-www-form-urlencoded'}self.form_data = {'fieldCount': 1,'action': 'search','product': 'WOS','search_mode': 'GeneralSearch','SID': sid,'max_field_count': 25,'formUpdated': 'true','value(input1)': kanming,'value(select1)': 'TI','value(hidInput1)': '','limitStatus': 'collapsed','ss_lemmatization': 'On','ss_spellchecking': 'Suggest','SinceLastVisit_UTC': '','SinceLastVisit_DATE': '','period': 'Range Selection','range': 'ALL','startYear': '1982','endYear': '2017','update_back2search_link_param': 'yes','ssStatus': 'display:none','ss_showsuggestions': 'ON','ss_query_language': 'auto','ss_numDefaultGeneralSearchFields': 1,'rs_sort_by': 'PY.D;LD.D;SO.A;VL.D;PG.A;AU.A'}self.form_data2 = {'product': 'WOS','prev_search_mode': 'CombineSearches','search_mode': 'CombineSearches','SID': sid,'action': 'remove','goToPageLoc': 'SearchHistoryTableBanner','currUrl': 'https://apps.webofknowledge.com/WOS_CombineSearches_input.do?SID=' + sid + '&product=WOS&search_mode=CombineSearches','x': 48,'y': 9,'dSet': 1}def craw(self, root_url,i):try:s = requests.Session()r = s.post(root_url, data=self.form_data, headers=self.hearders)r.encoding = r.apparent_encodingtree = etree.HTML(r.text)cited = tree.xpath("//div[@class='search-results-data-cite']/a/text()")download = tree.xpath(".//div[@class='alum_text']/span/text()")flag = 0print(i,cited, download,r.url)flag=0return cited, download, flagexcept Exception as e:if i == 0:print(e)print(i)flag = 1return cited, download, flagdef delete_history(self):murl = 'https://apps.webofknowledge.com/WOS_CombineSearches.do's = requests.Session()s.post(murl, data=self.form_data2, headers=self.hearders)root_url = 'https://apps.webofknowledge.com/WOS_GeneralSearch.do'if __name__ == "__main__":# sid='6AYLQ8ZFGGVXDTaCTV9'root = 'http://www.webofknowledge.com/'s = requests.get(root)sid = re.findall(r'SID=\w+&', s.url)[0].replace('SID=', '').replace('&', '')data = xlrd.open_workbook('2015年研究生发表论文.xlsx')table = data.sheets()[2]#具体是取哪个表格nrows = table.nrowsncols = table.ncolsctype = 1xf = 0for i in range(2, nrows):csv = open('2015_3.csv', 'a')fail = open('fail.txt', 'a')if i % 100 == 0:# 每一百次更换sids = requests.get(root)sid = re.findall(r'SID=\w+&', s.url)[0].replace('SID=', '').replace('&', '')kanming = table.cell(i, 5).value#取第i行第6列的数据obj_spider = SpiderMain(sid, kanming)cited,download,flag = obj_spider.craw(root_url,i)if flag==1:fail.write(str(i)+'\n')else:if len(cited)==0:cited.append(0)print(cited)if len(download)==0:download.append(0)download.append(0)print(download)csv.write(str(i) +  ',' + str(cited[0]) + ',' + str(download[0]) + ',' + str(download[1]) +'\n')csv.close()

title_wos_2.0

import re
# from threading import Thread
from multiprocessing import Process
from multiprocessing import Manager
import requests
import time
import xlrd
from bs4 import BeautifulSoup
from lxml import etreeclass SpiderMain(object):def __init__(self, sid, kanming):self.hearders = {'Origin': 'https://apps.webofknowledge.com','Referer': 'https://apps.webofknowledge.com/UA_GeneralSearch_input.do?product=UA&search_mode=GeneralSearch&SID=R1ZsJrXOFAcTqsL6uqh&preferencesSaved=','User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36",'Content-Type': 'application/x-www-form-urlencoded'}self.form_data = {'fieldCount': 1,'action': 'search','product': 'WOS','search_mode': 'GeneralSearch','SID': sid,'max_field_count': 25,'formUpdated': 'true','value(input1)': kanming,'value(select1)': 'TI','value(hidInput1)': '','limitStatus': 'collapsed','ss_lemmatization': 'On','ss_spellchecking': 'Suggest','SinceLastVisit_UTC': '','SinceLastVisit_DATE': '','period': 'Range Selection','range': 'ALL','startYear': '1982','endYear': '2017','update_back2search_link_param': 'yes','ssStatus': 'display:none','ss_showsuggestions': 'ON','ss_query_language': 'auto','ss_numDefaultGeneralSearchFields': 1,'rs_sort_by': 'PY.D;LD.D;SO.A;VL.D;PG.A;AU.A'}self.form_data2 = {'product': 'WOS','prev_search_mode': 'CombineSearches','search_mode': 'CombineSearches','SID': sid,'action': 'remove','goToPageLoc': 'SearchHistoryTableBanner','currUrl': 'https://apps.webofknowledge.com/WOS_CombineSearches_input.do?SID=' + sid + '&product=WOS&search_mode=CombineSearches','x': 48,'y': 9,'dSet': 1}def craw(self, root_url,i):try:s = requests.Session()r = s.post(root_url, data=self.form_data, headers=self.hearders)r.encoding = r.apparent_encodingtree = etree.HTML(r.text)cited = tree.xpath("//div[@class='search-results-data-cite']/a/text()")download = tree.xpath(".//div[@class='alum_text']/span/text()")flag = 0print(cited, download,r.url)flag=0return cited, download, flagexcept Exception as e:# 出现错误，再次try，以提高结果成功率if i == 0:print(e)print(i)flag = 1return cited, download, flagdef delete_history(self):murl = 'https://apps.webofknowledge.com/WOS_CombineSearches.do's = requests.Session()s.post(murl, data=self.form_data2, headers=self.hearders)class MyThread(Process):def __init__(self, sid, kanming, i, dic):Process.__init__(self)self.row = iself.sid = sidself.kanming = kanmingself.dic=dicdef run(self):self.cited, self.download, self.fl = SpiderMain(self.sid, self.kanming).craw(root_url, self.row)self.dic[str(self.row)]=Result(self.download, self.cited, self.fl, self.kanming, self.row)class Result():def __init__(self, download, cited, fl, kanming, row):super().__init__()self.row = rowself.kanming = kanmingself.fl = flself.cited = citedself.download = downloaddef runn(sid, kanming, i, d):ar, ref, fl = SpiderMain(sid, kanming).craw(root_url, row)d[str(i)]=Result(ar, ref, fl, kanming, i)print(d)
root_url = 'https://apps.webofknowledge.com/WOS_GeneralSearch.do'if __name__ == "__main__":# sid='6AYLQ8ZFGGVXDTaCTV9'root = 'http://www.webofknowledge.com/'s = requests.get(root)sid = re.findall(r'SID=\w+&', s.url)[0].replace('SID=', '').replace('&', '')data = xlrd.open_workbook('2015年研究生发表论文.xlsx')table = data.sheets()[0]nrows = table.nrowsncols = table.ncolsctype = 1xf = 0threads = []threadnum = 5d = Manager().dict()csv = open('2015_3.csv', 'a')fail = open('fail2015.txt', 'a')for i in range(2, nrows):if i % 100 == 0:# 每一百次更换sids = requests.get(root)sid = re.findall(r'SID=\w+&', s.url)[0].replace('SID=', '').replace('&', '')kanming = table.cell(i, 5).valuet = MyThread(sid, kanming, i, d)threads.append(t)if i % threadnum == 0 or i == nrows - 1:for t in threads:try:t.daemon = Truet.start()except requests.exceptions.ReadTimeout:continuefor t in threads:t.join()for t in threads:rst = d[str(t.row)]cited,download,flag = rst.cited,rst.download,rst.flif flag==1:fail.write(str(i)+'\n')else:if len(cited)==0:cited.append(0)print(cited)if len(download)==0:download.append(0)download.append(0)print(download)csv.write(str(i) + ","  + str(cited[0]) + ',' + str(download[0]) + ',' + str(download[1]) +'\n')threads = []csv.close()fail.close()