此文主要记录我在18年寒假期间，收集Avrix论文的总结

寒假生活题外

  在寒假期间，爸妈每天让我每天跟着他们6点起床，一起吃早点收拾，每天7点也就都收拾差不多。
  早晨的时光是人最清醒的时刻，而且到十点左右才开始帮忙做中午饭，中间这么大把的时光，我就来做做自己喜欢的事情。小外甥女也回来，但她每天只有10点起床后才跟我玩，真希望她能早起背背古诗文。

概述

  整个项目由数据采集（Python）,数据存储（Mysql），数据可视化（C#）组成。
  数据采集主要负责从网络上，获取Avrix的论文基本信息与论文下载地址。将其存储至MySQL，此过程中将分析Avrix上网页的结构，依靠依然是Chrome进行。
  数据存储食用的MySQL,其实我蛮想用SqlServer的，之前的东家是SqlServer，操作极其稳定与简单，速度很快，功能很全。新东家节约成本给我们多发点工资，用起来MySql了，使用了一段时间，大体上差不多，数据转存备份比转存比SqlServer复杂一些。
  数据可视化，目前做的部分是C#编译的一个查询界面，可以查询到相应的论文，可以从数据库中检索出论文，并链接指向的PDF文件，使用系统默认软件打开PDF。

分项陈述

pyhon数据采集部分

  直接上代码

# -*- coding:utf-8 -*-#作者：Qt.chao
#时间：2018/02/20
#综述：从Cornell University的图书馆Arvix主页上，获取计算机领域Computer Vision and Pattern Recognition的相关论文import urllib.request
import pymysql
from bs4 import BeautifulSoup
import requests
import time
import re
import os# 数据库连接基础类
class Conn_Mssql:#查询Mysql使用sql语句def Select_mssql(strsql):#数据库连接信息conn = pymysql.connect("localhost", "**username**", "**password**", "internetdaq", charset="utf8")cur = conn.cursor()cur.execute(strsql)return cur#插入与更新sql语句使用def InsertOrUpdate_mssql(strsql):# 数据库连接信息conn = pymysql.connect("localhost", "**username**", "**password**", "internetdaq", charset="utf8")cur = conn.cursor()cur.execute(strsql)conn.commit()conn.close()return cur#获取网络信息中的信息，并存储
class Get_HttpMessage:# 下载文件函数（根据连接地址，下载至D:StorePDF目录下）def getFile(url):try:file_name = url.split('/')[-1]file_path = "StorePDF\\"+file_nameu = urllib.request.urlopen(url)except :print(url, "url file not found")returnblock_sz = 90192with open(file_path, 'wb') as f:while True:buffer = u.read(block_sz)if buffer:f.write(buffer)else:break# 成功获取下载并打印下载信息print("Sucessful to download" + " " + file_name)# 获取文章中的PDF文档链接地址并下载def getPaperFile(url,file_name,path):try:file_name = url.split('/')[-1]file_path = path +"\\"+file_nameu = urllib.request.urlopen(url)except :print(url, "url file not found")returnblock_sz = 901920with open(file_path, 'wb') as f:while True:buffer = u.read(block_sz)if buffer:f.write(buffer)else:breakprint("Sucessful to download" + " " + file_name)# 从页面中获取论文数据（PDF下载地址与论文的标题等信息）def startGet(strUrl):print('start')# 链接的APPM网络url = strUrlrequest = urllib.request.Request(url)response = urllib.request.urlopen(request)data = response.read()soup = BeautifulSoup(data, "lxml")for link1 in soup.find_all(id=re.compile("dlpage")):for linklist in link1.find_all("dl"):# 论文连接地址相关的信息linklistLpdf = linklist.find_all("dt")# 论文标题作者等相关信息linklistLName = linklist.find_all("dd")# 节点信息长度cont_pdf = len(linklistLpdf)cont_Name = len(linklistLName)if cont_pdf == cont_Name :for linkNum in range(0,(cont_Name)):onepdf = linklistLpdf[linkNum].find_all(href=re.compile("pdf"))if len(onepdf)>0 :# PDF下载的连接地址pdfurl ="https://arxiv.org"+ (onepdf[0])['href']+".pdf"# 文章的详细信息链接地址paper_DetailUrls = linklistLpdf[linkNum].find_all(href=re.compile("abs"))paper_DetailUrl = "https://arxiv.org" + (paper_DetailUrls[0])['href']# 包含论文题目，作者，摘要的节点oneNames = linklistLName[linkNum].find_all(class_="meta")# Paper的编号信息oneName = paper_DetailUrls[0].get_text().replace("'","^",999)# 文章的标题Paper_Titles = oneNames[0].find_all(class_="list-title mathjax")Paper_Title = Paper_Titles[0].get_text().replace("'","^",999)# 文章的作者Paper_Authors = oneNames[0].find_all(class_="list-authors")Paper_Author =""for Paper_Authorlist in Paper_Authors[0].find_all("a"):Paper_Author =Paper_Author +Paper_Authorlist.get_text()+"|"Paper_Author = Paper_Author.replace("'","^",999)Paper_Author = Paper_Author[:-1]# 获取摘要信息time.sleep(5)request2 = urllib.request.Request(paper_DetailUrl)response2 = urllib.request.urlopen(request2)data2 = response2.read()soup2= BeautifulSoup(data2, "lxml")Paper_ABSTRACTS = soup2.find_all(class_="abstract mathjax")if len(Paper_ABSTRACTS)>0 :Paper_ABSTRACT = Paper_ABSTRACTS[0].get_text().replace("'", "^", 999)else:Paper_ABSTRACT = ""# 分类信息PAPER_SUBJECT = ""PAPER_SUBJECTs = soup2.find_all(class_="tablecell subjects")if len(PAPER_SUBJECTs)>0 :PAPER_SUBJECT = PAPER_SUBJECTs[0].get_text().replace("'", "^", 999)else:PAPER_SUBJECT = ""# 获取网页的详细内容信息Paper_Detail = soup2.prettify().replace("'","^",999)Paper_Detail = Paper_Detail.replace("-->","  -->",999)strSQL= "CALL SaveTheCornellUniversityPaper(0,'"+oneName+"','" + Paper_Detail + "','"+pdfurl+"',0,'" + Paper_Title + "','" + Paper_Author + "','" + Paper_ABSTRACT + "','" + PAPER_SUBJECT + "')"strSQL = strSQL.encode('utf8')try:# 存储地址信息Conn_Mssql.InsertOrUpdate_mssql(strSQL)# Get_HttpMessage.getFile(pdfurl)time.sleep(4)print('母页面MySQL存储成功')except:print(strSQL)print('母页面MySQL存储失败')time.sleep(5)# 从主界面获取主界面与分界面的连接信息def selectUrl(strUrl):print(strUrl)Get_HttpMessage.startGet(strUrl)request = urllib.request.Request(strUrl)response = urllib.request.urlopen(request)data = response.read()soup = BeautifulSoup(data, "lxml")for link1 in soup.find_all(id=re.compile("dlpage")):linklists = link1.find_all("small")for linkUrl in linklists[0].find_all("a"):oneUrl = "https://arxiv.org" + linkUrl["href"]print(oneUrl)Get_HttpMessage.startGet(oneUrl)# 从数据库中获取链接地址，下载论文def DownlooadPDFFromMysql(str= ""):urlRows = Conn_Mssql.Select_mssql("SELECT TILE_NAME ,URL from cornell_paper WHERE FLAGE = 0 order BY UID DESC  ;")for urlRow in urlRows:try:Get_HttpMessage.getPaperFile(urlRow[1],urlRow[0]+".pdf","D:\\CornellLibrary")urlRowfiles =urlRow[0].split(":")urlRowfile = urlRowfiles[1]charFILE_PATH = "D:\\\\CornellLibrary\\\\"+urlRowfile+".pdf"strsql = "UPDATE cornell_paper SET FLAGE = 1 , FILE_PATH = '"+ charFILE_PATH +"' WHERE URL = '"+urlRow[1]+"'"Conn_Mssql.InsertOrUpdate_mssql(strsql)print("下载成功"+ charFILE_PATH )except:print("下载失败")finally:time.sleep(3)
# 程序入口
# Arvix相关主页的链接地址
MainUrl = '网络地址'
#  从网页中抓取相关数据信息
Get_HttpMessage.selectUrl(MainUrl)
# 从MySQL数据库中下载论文
Get_HttpMessage.DownlooadPDFFromMysql()

如上即我采集Arvix主页上Computer Vision and Pattern Recognition相关信息信相关信息信，先将网页信息逐个解析，分类存储。将网页的信息存储后，从数据库中，按照标志位，根据下载地址信息，下载相应论文。

MySQL数据存储部分

在整个采集的过程中，是将数据存储在一数据库的一张表中，数据库名称为internetdaq，表（cornell_paper）的结构是：

数据库表结构SQL语句为：

DROP TABLE IF EXISTS `cornell_paper`;
CREATE TABLE `cornell_paper` (`UID` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '自增无重复编号',`SAVE_TIME` datetime DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '保存时间',`TILE_NAME` varchar(200) COLLATE utf8_unicode_ci DEFAULT NULL COMMENT '标题名称',`TYPE` int(11) DEFAULT NULL COMMENT '存储类型',`PAGE_DETAIL` text COLLATE utf8_unicode_ci COMMENT '网页内详细信息',`FLAGE` int(11) DEFAULT NULL COMMENT '标志位',`URL` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL COMMENT '文章下载连接地址',`PAPER_TITLE` varchar(1500) COLLATE utf8_unicode_ci DEFAULT NULL COMMENT '文章题目',`PAPER_AUTHOR` varchar(500) COLLATE utf8_unicode_ci DEFAULT NULL COMMENT '文章作者',`PAPER_ABSTRACT` varchar(6000) COLLATE utf8_unicode_ci DEFAULT NULL COMMENT '文章摘要',`PAPER_SUBJECT` varchar(500) COLLATE utf8_unicode_ci DEFAULT NULL COMMENT '文章分类',`FILE_PATH` varchar(300) COLLATE utf8_unicode_ci DEFAULT NULL COMMENT '文件路径',PRIMARY KEY (`UID`)
) ENGINE=InnoDB AUTO_INCREMENT=2445 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

数据库使用的存储结构SQL语句为：

DROP PROCEDURE IF EXISTS `SaveTheCornellUniversityPaper`;
CREATE PROCEDURE SaveTheCornellUniversityPaper(in int_Type int, in CharTILE_NAME varchar(200),
in CharPAGE_DETAIL text,in CharURL varchar(254),in int_Flage int
,in CharPAPER_TITLE varchar(1500)
,in CharPAPER_AUTHOR varchar(500)
,in CharPAPER_ABSTRACT varchar(6000)
,in CharPAPER_SUBJECT varchar(500)
)
MODIFIES SQL DATA
COMMENT'从Cornell大学的论文主页获取论文的信息并存储 int_Type(类型) CharTILE_NAME(标题) CharPAGE_DETAIL(网页信息) CharURL(链接地址) int_Flage(标志位)'
BEGINDECLARE pdfCount int DEFAULT 0;SELECT COUNT(URL) INTO pdfCount from cornell_paper WHERE URL = CharURL;
IF pdfCount< 1 THENINSERT INTO cornell_paper(SAVE_TIME,TYPE,TILE_NAME,PAGE_DETAIL,FLAGE,URL,PAPER_TITLE,PAPER_AUTHOR,PAPER_ABSTRACT,PAPER_SUBJECT)VALUES(NOW(),int_Type,CharTILE_NAME,CharPAGE_DETAIL,int_Flage,CharURL,CharPAPER_TITLE,CharPAPER_AUTHOR,CharPAPER_ABSTRACT,CharPAPER_SUBJECT);
END IF;END

CSharp数据查询部分

此部分主要是用C#从数据库中提取数据，并查询连接查看，主要是为了平时查看论文的具体内容和PDF文档，用于实现的代码也较为简单，没做过多修饰，在接下完善中，可能使用python或C#进行进一步的语义分析与数据可视化，目前主要逻辑如何下所示

主查询界面

using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Data;
using System.Windows.Documents;
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Navigation;
using System.Windows.Shapes;
using LCS;namespace PGCL
{/// <summary>/// MainWindow.xaml 的交互逻辑/// </summary>public partial class MainWindow : Window{public MainWindow(){InitializeComponent();}private DataTable dataGridDataTable;/// <summary>/// 窗体登录/// </summary>/// <param name="sender"></param>/// <param name="e"></param>private void MainWindow_OnLoaded(object sender, RoutedEventArgs e){}/// <summary>/// 刷新按钮/// </summary>/// <param name="sender"></param>/// <param name="e"></param>private void BtnUpdate_OnClick(object sender, RoutedEventArgs e){ShowAllDate();}/// <summary>/// 刷新显示所有数据列表/// </summary>private void ShowAllDate(){DataTable myDataTable = LCS_Lib_DataMySql.Mysql_SelectDataTable("SELECT SAVE_TIME,PAPER_TITLE,PAPER_SUBJECT,URL FROM `cornell_paper` ORDER BY UID DESC ;");dataGridDataTable = new DataTable();dataGridDataTable.Columns.Add("Num", typeof(string));dataGridDataTable.Columns.Add("Save_time", typeof(string));dataGridDataTable.Columns.Add("Title", typeof(string));dataGridDataTable.Columns.Add("Subject", typeof(string));dataGridDataTable.Columns.Add("URL", typeof(string));int IntNum = 1;foreach (DataRow VARDataRow in myDataTable.Rows){DataRow oneDataRow = dataGridDataTable.NewRow();oneDataRow["Num"] = IntNum.ToString();oneDataRow["Save_time"] = VARDataRow["SAVE_TIME"].ToString();oneDataRow["Title"] = VARDataRow["PAPER_TITLE"].ToString();oneDataRow["Subject"] = VARDataRow["PAPER_SUBJECT"].ToString();oneDataRow["URL"] = VARDataRow["URL"].ToString();dataGridDataTable.Rows.Add(oneDataRow);IntNum++;}Paper_datagrid.ItemsSource = dataGridDataTable.DefaultView;}/// <summary>/// 双击表格/// </summary>/// <param name="sender"></param>/// <param name="e"></param>private void Paper_datagrid_OnMouseDoubleClick(object sender, MouseButtonEventArgs e){DataRowView onreRowView = (DataRowView) Paper_datagrid.SelectedItem;DataRow oneDataRow = onreRowView.Row;DataTable myDataTable = LCS_Lib_DataMySql.Mysql_SelectDataTable("SELECT PAPER_TITLE,PAPER_AUTHOR,PAPER_SUBJECT,PAGE_DETAIL,FILE_PATH" +" FROM cornell_paper WHERE URL = '" + oneDataRow["URL"].ToString() + "' ");winDetail myWinDetail = new winDetail();myWinDetail.strTile = myDataTable.Rows[0]["PAPER_TITLE"].ToString();myWinDetail.strAuthor = myDataTable.Rows[0]["PAPER_AUTHOR"].ToString();myWinDetail.strSubject = myDataTable.Rows[0]["PAPER_SUBJECT"].ToString();myWinDetail.strpage_detail = myDataTable.Rows[0]["PAGE_DETAIL"].ToString();string[] arelinkStrings = myDataTable.Rows[0]["FILE_PATH"].ToString().Split(':');string strLkink = "";if (myDataTable.Rows[0]["FILE_PATH"].ToString().Length>0){strLkink = myDataTable.Rows[0]["FILE_PATH"].ToString();}else{strLkink = "";}myWinDetail.strPDFlink = strLkink;myWinDetail.ShowDialog();}}
}

详细信息查询

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Data;
using System.Windows.Documents;
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Navigation;
using System.Windows.Shapes;namespace PGCL
{/// <summary>/// winDetail.xaml 的交互逻辑/// </summary>public partial class winDetail : Window{public winDetail(){InitializeComponent();}/// <summary>/// 题目/// </summary>public string strTile { set; get; }/// <summary>/// 作者/// </summary>public string strAuthor { set; get; }/// <summary>/// 分类标签/// </summary>public string strSubject { set; get; }/// <summary>/// 网页详细信息/// </summary>public string strpage_detail { set; get; }/// <summary>/// PDF连接地址/// </summary>public string strPDFlink { set; get; }/// <summary>/// 关闭按钮/// </summary>/// <param name="sender"></param>/// <param name="e"></param>private void Btn_close_OnClick(object sender, RoutedEventArgs e){this.Close();}/// <summary>/// 窗体登录/// </summary>/// <param name="sender"></param>/// <param name="e"></param>private void WinDetail_OnLoaded(object sender, RoutedEventArgs e){lab_tile.Content = strTile;lab_author.Content = strAuthor;lab_subject.Content = strSubject;web_detail.NavigateToString(strpage_detail);lab_link.Content = strPDFlink;lab_link.Foreground = new SolidColorBrush(Colors.Blue);}public void SuppressScriptErrors(WebBrowser wb, bool Hide){FieldInfo fiComWebBrowser = typeof(WebBrowser).GetField("_axIWebBrowser2", BindingFlags.Instance | BindingFlags.NonPublic);if (fiComWebBrowser == null) return;object objComWebBrowser = fiComWebBrowser.GetValue(wb);if (objComWebBrowser == null) return;objComWebBrowser.GetType().InvokeMember("Silent", BindingFlags.SetProperty, null, objComWebBrowser, new object[] { Hide });}private void Web_detail_OnNavigated(object sender, NavigationEventArgs e){SuppressScriptErrors(web_detail, true);}/// <summary>/// 双击表格/// </summary>/// <param name="sender"></param>/// <param name="e"></param>private void Lab_link_OnMouseDoubleClick(object sender, MouseButtonEventArgs e){Process MyProcess = new Process();MyProcess.StartInfo.FileName = lab_link.Content.ToString();MyProcess.StartInfo.Verb = "Open";MyProcess.StartInfo.CreateNoWindow = true;MyProcess.Start();}}
}

总结

上述即为我这次数据采集的实践，再接下来的学习中，我会向着词频分析与数据可视化方向进一步拓展。上述代码不能直接复制粘贴使用，仅供参考。