这里使用nodejs下的chrome-har库来导出浏览器的har数据，经验证效果不错，比较靠谱。

1，创建日志配置(ultra-harlog/module/log.js)

//cnpm install --save log4js
const log4js = require('log4js');const options = {appenders:{console:{type: "console"},"puppeteer-record":{type : 'dateFile',filename : 'logs/puppeteer/log',pattern : '-yyyy-MM-dd.log',alwaysIncludePattern : true,encoding : 'utf-8'},"puppeteer-har-record":{type : 'dateFile',filename : 'logs/puppeteerhar/log',pattern : '-yyyy-MM-dd.log',alwaysIncludePattern : true,encoding : 'utf-8'},"puppeteer-harevent-record":{type : 'dateFile',filename : 'logs/puppeteerharevent/log',pattern : '-yyyy-MM-dd.log',alwaysIncludePattern : true,encoding : 'utf-8'}}   ,"categories": {"default": { "appenders": ['console', "puppeteer-record", "puppeteer-har-record","puppeteer-harevent-record"], "level": "all" }}
}
log4js.configure(options);function getConsoleLogger(){let consoleLog = log4js.getLogger('console');   return consoleLog ;
}function getPuppeteerRecordLogger(){let consoleLog = log4js.getLogger('puppeteer-record');  return consoleLog ;
}function getPuppeteerHarRecordLogger(){let consoleLog = log4js.getLogger('puppeteer-har-record');  return consoleLog ;
}function getPuppeteerHarEventRecordLogger(){let consoleLog = log4js.getLogger('puppeteer-harevent-record'); return consoleLog ;
}exports.getConsoleLogger = getConsoleLogger;
exports.getPuppeteerRecordLogger = getPuppeteerRecordLogger;
exports.getPuppeteerHarRecordLogger = getPuppeteerHarRecordLogger;
exports.getPuppeteerHarEventRecordLogger = getPuppeteerHarEventRecordLogger;

创建抓取的代码(harlog/module/puppeteerhar.js)

const puppeteer = require('puppeteer');
const PuppeteerHar = require('puppeteer-har');
const path = require("path");const logger=require("./log");
const grpcclient=require("./grpcclient");const log = logger.getPuppeteerHarRecordLogger() ;/*启动浏览器
*/
async function launchBrowser(){//启动浏览器实例 [puppeteer.createBrowserFetcher([options])]let browser = await puppeteer.launch({// 若是手动下载的chromium需要指定chromium地址, 默认引用地址为 /项目目录/node_modules/puppeteer/.local-chromium///executablePath: '/Users/huqiyang/Documents/project/z/chromium/Chromium.app/Contents/MacOS/Chromium',//如果是访问https页面 此属性会忽略https错误ignoreHTTPSErrors: true,// 关闭headless模式, 不会打开浏览器headless: true,//浏览器启动参数 https://peter.sh/experiments/chromium-command-line-switches/   --timeoutargs:['--disk-cache-size=0','--disable-cache','--disable-infobars','--window-size=800,600','--ignore-certificate-errors','--enable-feaures'],//是否为每个选项卡自动打开DevTools面板。 如果此选项为true，则headless选项将设置为false。devtools: false,//Defaults to 30000 (30 seconds). Pass 0 to disable timeout.timeout: 0//放慢puppeteer执行的动作,方便调试//slowMo: 250});return browser ;
}async function saveHarlog(url,dirPath,filename){let homesite = url ;//保存的文件路径let harFilePath = path.join(dirPath,filename) ;//处理URLif(!(url.startsWith('http://') || url.startsWith('https://'))){url = "http://" + url ;}//打开浏览器let browser = await launchBrowser() ;//Puppeteer 初始化的屏幕大小默认为 800px x 600px。但是这个尺寸可以通过 Page.setViewport() 设置。/*await page.setViewport({width: 800,height: 600});*///创建一个新页面//let page = await browser.newPage();const page = (await browser.pages())[0];await page.waitFor(1000); //delay 1 s//page.setDefaultTimeout(12000);//page.setJavaScriptEnabled(enabled)//事件监听轻松打出页面的log//page.on('console', msg => log.info('PAGE LOG:', msg.text()));let har = new PuppeteerHar(page);try{await har.start({ path:harFilePath});/*页面跳转相关函数：page.goto(url, options)  //相当于在浏览器中输入了地址，然后回车page.goBack(options)page.goForward(options)page.reload(options)*/await page.goto(url,{timeout:0});log.info(page.mainFrame().title());log.info(page.mainFrame().url());//返回HTML文档内容//const html = await page.$eval('html', e => e.outerHTML);//const html = await page.content() ;//通知JAVA解析HAR文件/*try{grpcclient.resovleHarLog({url:homesite,file_name:filename,file_dir:dirPath,context:''});}catch(err){log.error('发送RPC请求失败,' + err);}*/}catch(error){log.info('resovle error :' + url + ";  error message:" + error) ;}finally{if(har){await har.stop();       }if(browser){await browser.close();      }}
}
exports.launchBrowser = launchBrowser;
exports.saveHarlog = saveHarlog;

创建启动文件(ultra-harlog/puppeteerhar-app.js)

const fs = require("fs");
const path = require("path");
const moment = require("moment");
const schedule = require('node-schedule');const cvsresovler=require("./module/cvsresovle");
const mhar=require("./module/puppeteerhar");/*
cnpm install --save moment
cnpm install --save csv
cnpm install --save node-schedule
cnpm install --save puppeteer
cnpm install --save puppeteer-har
cnpm install --save iconv-lite
cnpm install --save chrome-harcnpm install --save grpc*/
function init(){console.log('初始化调度器') ;//每分钟的第30秒定时执行一次:schedule.scheduleJob('0 14 10 * * *',()=>{let ftime = moment().format('YYYYMMDDHHmm');console.log('当前调度时间为:' + ftime) ;let dirPath = path.join(__dirname,'harlogs',ftime) ;console.log("创建目录:" + dirPath) ;let isExist = false ;if(fs.existsSync(dirPath)){//创建文件夹let stat = fs.lstatSync(dirPath);if(stat.isDirectory()){isExist = true ;}}if(!isExist){//创建文件夹console.log("创建文件夹" + ftime) ;fs.mkdirSync(dirPath);}       //开始解析需要处理的URLlet dataArr = cvsresovler.readUrlRecord(path.join(__dirname,'top300.csv')) ;console.log("解析出URL共计" + dataArr.length + "条") ;/*开始抓取HAR数据【同步的方式执行】。注意：如果这里直接通过for循环遍历dataArr并调用saveHarlog方法，那么这将是一个异步的过程。*/(async function iterator(i){let data =  dataArr[i]let url = data['SITE_LINK'] ;url = url.trim() ;let filename = url.replace(/\//g,'-').replace(/\\/g,'-') + '.har' ;if(url){console.log((i+1) + "-starting to resovle url ：" + url ) ;try{await mhar.saveHarlog(url,dirPath,"N" + "-" + filename) ;}catch(error){console.log(error) ;}}if(i + 1 < dataArr.length){iterator(i+1) ;}})(0) ;}); console.log('应用程序启动完成') ;
}
//执行
//init();/**用于测试的方法
*/
async function test(){let ftime = moment().format('YYYYMMDDHHmm');console.log('当前执行时间为:' + ftime) ;let dirPath = path.join(__dirname,'harlogs',ftime) ;console.log("创建目录:" + dirPath) ;let isExist = false ;if(fs.existsSync(dirPath)){//创建文件夹let stat = fs.lstatSync(dirPath);if(stat.isDirectory()){isExist = true ;}}if(!isExist){//创建文件夹console.log("创建文件夹" + ftime) ;fs.mkdirSync(dirPath);}     //测试的URLlet url = "www.baidu.com" ; let arguments = process.argv.splice(2);if(arguments.length > 0 ){url = arguments[0] ;}url = url.trim() ;let filename = url.replace(/\//g,'-').replace(/\\/g,'-') + '.har' ;if(url){console.log("starting to resovle test url ：" + url ) ;try{await mhar.saveHarlog(url,dirPath,"NT" + "-" + filename) ;}catch(error){console.log(error) ;}}
}
//运行测试
test() ;

关于GRPC部分的代码，请参考我另外一篇博文

参考地址：https://michaljanaszek.com/blog/generate-har-with-puppeteer