function get_url_array(start, end) {var url_template = "http://stockdata.stock.hexun.com/ggzjc/data/ChangeHistory.aspx?count=1000&page=%d&callback=hxbase_json5"var util = require("util")var array = new Array()for (var i = start; i <= end; i++) {var url_one = util.format(url_template, i + 1)array.push(url_one)}return array
}

function get_data_from_url(url) {var request = require('sync-request');var user_agent_list =['Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36','Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1','Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50','Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1','Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11','Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56',"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 ","Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 ","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 ","Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 ","Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 ","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 ","Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 ","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 ","Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 ","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 ","Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 ","Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 ","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 ","Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 ","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36"]var pos =randomIntRange(0,user_agent_list.length-1)// 增加user-agentvar res = request('GET', url, {'headers': {'user-agent':user_agent_list[pos],'Host': 'stockdata.stock.hexun.com','Referer': 'http://stockdata.stock.hexun.com/ggzjc/history.shtml'},retry : true,retryDelay: 10000,maxRetries: 5,timeout:200000});var buf = res.getBody()var iconv = require("iconv-lite")// 使用gb2312编码方式var data_str = iconv.decode(buf, 'gb2312')data_str = data_str.replace(/上海市浦东新区公共交通投资发\\/g,"上海市浦东新区公共交通投资发")var data_list = eval(data_str)return data_list.list
}

//wrong !
changePeopleTitle: '上海市浦东新区公共交通投资发\'
//right
changePeopleTitle: '上海市浦东新区公共交通投资发'

//随机生成范围在low,high之间的随机数
function randomIntRange (low, high) {return Math.floor(Math.random() * (high - low + 1) + low);
}

function hxbase_json5(str) {var data = eval(str)return data
}

function save_data_to_mysql() {var Sequelize = require('sequelize')var sleep = require("sleep")var sequelize = new Sequelize('dbname','root','passwd',{'dialect': 'mysql','host': '127.0.0.1','port': 3306,define: {charset: 'utf8',timestamps: false//不定义时间戳
            }})//高管增减持var Ggzjc = sequelize.define('table_name', {'stock_code': {//股票代码'type': Sequelize.STRING,'allowNull': false,'unique': false},'stock_name': {//股票名称'type': Sequelize.STRING,'allowNull': false,'unique': false},'changeDate': {//变动日期'type': Sequelize.DATEONLY,'allowNull': true},'noticeDate': {//公告日期'type': Sequelize.DATEONLY,'allowNull': true},'changeNum': {// 变动数量 万股'type': Sequelize.DOUBLE,'allowNull': true},'averagePrice': {//均价'type': Sequelize.DOUBLE,'allowNull': true},'price': {//金额'type': Sequelize.DOUBLE,'allowNull': true},'shareHoldingNum': {//变动后持股数目'type': Sequelize.DOUBLE,'allowNull': true},'changeRatio': {//变动人变动比'type': Sequelize.DOUBLE,'allowNull': true},'circulationCapitalRatio': {//占流通股本比例'type': Sequelize.DOUBLE,'allowNull': true},'changeWay': {//变动方式'type': Sequelize.STRING,'allowNull': true},'changePeople': {//股份变动人'type': Sequelize.STRING,'allowNull': true},'changePeopleTitle': {//相关董事高管'type': Sequelize.STRING,'allowNull': true},'duties': {//职务'type': Sequelize.STRING,'allowNull': true},'relation': {//关系'type': Sequelize.STRING,'allowNull': true},'industry': {//行业'type': Sequelize.STRING,'allowNull': true},})Ggzjc.sync({force: true}).then(function () {var url_array = get_url_array(1, 58)for (var i = 0; i < url_array.length; i++) {var data = get_data_from_url(url_array[i])sleep.usleep(200000)print(i + 1)print('complete!')for (var j = 0; j < data.length; j++) {var changeDate = '20' + data[j].changeDatevar noticeDate = '20' + data[j].noticeDatevar str_array = data[j].stockName.split("(")var stock_name = str_array[0]str_array = str_array[1].split(")")var stock_code = str_array[0]var changeNum = get_content_from_html(data[j].changeNum)if (changeNum != null) {if (changeNum == "")changeNum = nullelsechangeNum = parseFloat(changeNum)}var averagePrice = data[j].averagePriceif (averagePrice == '&nbsp;')averagePrice = nullelseaveragePrice = parseFloat(averagePrice)var price = get_content_from_html(data[j].price)if (price != null) {if (price == '')price = nullelseprice = parseFloat(price)}var shareHoldingNum = data[j].shareHoldingNumif (shareHoldingNum == '&nbsp;')shareHoldingNum = nullelseshareHoldingNum = parseFloat(shareHoldingNum)var changeRatio = data[j].changeRatioif (changeRatio == '&nbsp;')changeRatio = nullelsechangeRatio = parseFloat(changeRatio)var circulationCapitalRatio = data[j].circulationCapitalRatioif (circulationCapitalRatio == '&nbsp;')circulationCapitalRatio = nullelsecirculationCapitalRatio = parseFloat(circulationCapitalRatio)var changeWay = data[j].changeWayif (changeWay == '&nbsp;')changeWay = nullvar changePeople = data[j].changePeople// console.log(data[j].changePeople)if (changePeople == '&nbsp;')changePeople = nullvar changePeopleTitle = data[j].changePeopleTitleif (changePeopleTitle == '&nbsp;')changePeopleTitle = nullvar duties = get_content_from_html(data[j].duties)if (duties != null && duties == '')duties = nullvar relation = get_content_from_html(data[j].relation)if (relation != null && relation == '')relation = nullvar industry = data[j].industryif (industry == '&nbsp;')industry = nullvar one = Ggzjc.build({'stock_code': stock_code,'stock_name': stock_name,'changeDate': changeDate,'noticeDate': noticeDate,'changeNum': changeNum,'averagePrice': averagePrice,'price': price,'shareHoldingNum': shareHoldingNum,'changeRatio': changeRatio,'circulationCapitalRatio': circulationCapitalRatio,'changeWay': changeWay,'changePeople': changePeople,'changePeopleTitle': changePeopleTitle,'duties': duties,'relation': relation,'industry': industry})one.save()}}})
}

function get_content_from_html(str) {var pattern = />[\s\S]+?</gvar res = str.match(pattern)if (res == null) {return null}var result = res[0]return result.slice(1, result.length - 1)
}