以编程方式抓取中国联通电子发票

电子发票通常是以pdf文件存在，各大电商几乎都提供电子发票，如京东、淘宝（天猫）、苏宁易购、携程、中国联通、电信、移动等平台。那么，我们如何以编程方式爬取这些平台的电子发票呢？好了，这里我直接上代码供参考，实际上，经测试，各大电商平台的电子发票都是可以爬取的。欢迎加qq283335746共同探讨以编程方式爬虫抓取数据信息。

/// <summary>/// 中国联通（10010.com）/// </summary>public class ChinaUnicom{private ChinaUnicom() { }public ChinaUnicom(string userId, string cookieAppend){this._userId = userId;this._userCookie = cookieAppend;this._relationUserId = Common.CookieFilter(_userCookie, string.Format(@"\s*{0}\s*=\s*(.|\n)*?;", "_uop_id"));this.OrderInvoices = new List<UnicomOrderInvoiceInfo>();this._invoiceBll = new ThirdPartyBll(userId, _relationUserId, ThirdPartyOptions.ChinaUnicom);this._thirdPartyRequest = new ThirdPartyRequest(cookieAppend);this._client = new NetClient(BaseUrl);}private const string BaseUrl = "http://wap.10010.com";private const string LoginPartialUrl = "uac.10010.com/oauth2/new_auth";private const string QueryInvoicesRefererUrl = "http://wap.10010.com/mobileService/query/einvoice.htm?navUrlCode=1307&menuId=000200060012";private const string QueryInvoicesUrl = "http://wap.10010.com/mobileService/query/einvoicelist.htm";private readonly string _userId = string.Empty;private readonly string _userCookie = string.Empty;private readonly string _relationUserId = string.Empty;private readonly NetClient _client;private readonly ThirdPartyBll _invoiceBll;private readonly ThirdPartyRequest _thirdPartyRequest;private List<UnicomOrderInvoiceInfo> OrderInvoices { get; set; }/// <summary>/// 发票处理入口/// </summary>/// <returns></returns>public async Task ExecuteInvoiceAsync(){var request = _thirdPartyRequest.CreateRequest(QueryInvoicesUrl, MethodOptions.Post);request.AddParameter(Dcr.ContentTypeKey, Dcr.ContentType, ParameterType.HttpHeader);//获取所有包含发票相关信息的订单集await GetOrderInvoicesAsync(request, 1, 40, "record");if (!OrderInvoices.Any()){if (Log.IsDebugEnabled) Log.Debug("ChinaUnicom.ExecuteInvoiceAsync,UserId:{0},RelationUserId:{1},not any invoice info! data from OrderInvoices.", _userId, _relationUserId);return;}var unicomInvoices = ToUnicomInvoices(OrderInvoices);if (unicomInvoices == null || !unicomInvoices.Any()){if (Log.IsDebugEnabled) Log.Debug("ChinaUnicom.ExecuteInvoiceAsync,not any invoice info! data from unicomInvoices.");return;}var oldInvoices = await _invoiceBll.GetUserThirdPartyTInvoices();foreach (var item in unicomInvoices){var filePath = await DownloadInvoiceAsync(item);if (string.IsNullOrEmpty(filePath)) continue;var oldInfo = oldInvoices?.FirstOrDefault(m => m.InvoiceCode == item.invoicecode && m.InvoiceNumber == item.invoicenum);if (oldInfo != null) continue;//await invoiceBll.SaveThirdPartyInvoice(item.busiorder, item.invoiceurl, filePath);await _invoiceBll.SaveInvoice(filePath, string.Empty);}}/// <summary>/// 获取包含发票相关信息的订单集/// </summary>/// <param name="request"></param>/// <param name="pageIndex"></param>/// <param name="pageSize"></param>/// <param name="type"></param>/// <returns></returns>private async Task GetOrderInvoicesAsync(NetRequest request, int pageIndex, int pageSize, string type){request.AddParameter("page", pageIndex.ToString());request.AddParameter("pageSize", pageSize.ToString());request.AddParameter("type", type);var response = await _client.ExecuteAsync(request);if (response.ResponseUri.ToString().Contains(LoginPartialUrl))throw new CustomException(MC.M_ThirdPartySiteLoginNeeding);if (string.IsNullOrEmpty(response.Content)) return;var orderInvoiceInfo = JsonConvert.DeserializeObject<UnicomOrderInvoiceInfo>(response.Content);if (orderInvoiceInfo?.totalmap == null || !orderInvoiceInfo.totalmap.Any()){if (Log.IsDebugEnabled) Log.Debug("ChinaUnicom.GetOrderInvoicesAsync,not has invoiceinfo! request Resource is : {0}", request.Resource);return;}OrderInvoices.Add(orderInvoiceInfo);}/// <summary>/// 下载并保存电子发票PDF文件，返回文件存储的物理路径/// </summary>/// <param name="invoiceInfo"></param>/// <returns></returns>private async Task<string> DownloadInvoiceAsync(UnicomInvoiceInfo invoiceInfo){if (string.IsNullOrEmpty(invoiceInfo.invoicecode) || string.IsNullOrEmpty(invoiceInfo.invoicenum)) return string.Empty;var filePath = FileHelper.GetFilePath("ChinaUnicom", string.Format(@"{0}-{1}.pdf", invoiceInfo.invoicecode, invoiceInfo.invoicenum));if (File.Exists(filePath)) return filePath;var request = _thirdPartyRequest.CreateRequest(invoiceInfo.invoiceurl, QueryInvoicesUrl);var response = await _client.ExecuteAsync(request);if (response.ContentLength == 0 || response.ContentType != Dcr.PdfContentType) return string.Empty;await FileHelper.Save(response.RawBytes, filePath);return filePath;}/// <summary>/// 将订单发票信息集过滤出发票信息集，过滤无用数据/// </summary>/// <param name="unicomOrderInvoices"></param>/// <returns></returns>private static IEnumerable<UnicomInvoiceInfo> ToUnicomInvoices(IEnumerable<UnicomOrderInvoiceInfo> unicomOrderInvoices){var unicomInvoices = from r in unicomOrderInvoiceslet t = r.totalmapfrom v in tlet invs = v.valuefrom inv in invsselect inv;return unicomInvoices;}}

以编程方式抓取中国联通电子发票相关推荐

以编程方式抓取京东商城电子发票
电子发票通常是以pdf文件存在,各大电商几乎都提供电子发票,如京东.淘宝(天猫).苏宁易购.携程.中国联通.电信.移动等平台.那么,我们如何以编程方式爬取这些平台的电子发票呢?好了,这里我直接上代码供 ...
#.NET分别以GET和POST方式抓取远程页面
代码引入命名空间 using System.IO; using System.Net; using System.Text; using System.Text.RegularExpressions ...
python3 爬虫第二步Selenium 使用简单的方式抓取复杂的页面信息
Selenium 简介该系列专栏上一篇爬虫文章点击这里. 网站复杂度增加,爬虫编写的方式也会随着增加.使用Selenium 可以通过简单的方式抓取复杂的网站页面,得到想要的信息. Selenium ...
java getconnection_java学习-GET方式抓取网页(UrlConnection和HttpClient)
抓取网页其实就是模拟客户端(PC端,手机端...)发送请求,获得响应数据documentation,解析对应数据的过程.---自己理解,错误请告知一般常用请求方式有GET,POST,HEAD三种 G ...
实例：使用puppeteer headless方式抓取JS网页
puppeteer google chrome团队出品的puppeteer 是依赖nodejs和chromium的自动化测试库,它的最大优点就是可以处理网页中的动态内容,如JavaScript,能够更 ...
python find()效率_基于python分别采用同步与异步（协程）方式抓取时光网TOP100电影...
欢迎各位小哥哥小姐姐阅读本的文章,对大家学习有帮助,请点赞加关注哦!!!!!!!!!! 您的点赞和关注将是我持续更新的动力呢.^v^ 有不懂的问题可以私聊我哦! 如题,同步运行就是python按照代码 ...
Python笔记：使用代理方式抓取App的数据操作案例
概述 App的爬取相比Web端爬取更加容易,反爬中能力没有那么强,而且响应数据大多都是JSON形式,解析更加简单. 在APP端若想查看和分析内容那就需要借助抓包软件,常用的有:Filddler.Cha ...
Python爬虫方式抓取免费http代理IP
我们新手在练手的时候,常常需要一些代理IP进行爬虫抓取,但是因为学习阶段,对IP质量要求不高,主要是弄懂原理,所以花钱购买爬虫代理IP就显得没必要(大款忽略),今天跟大家分享一下,如果使用爬虫抓取免费 ...
C# 爬虫正则、NSoup、HtmlAgilityPack、Jumony四种方式抓取小说
心血来潮,想爬点小说.通过百度选择了个小说网站,随便找了一本小说http://www.23us.so/files/article/html/13/13655/index.html. 1.分析html规 ...

以编程方式抓取中国联通电子发票

以编程方式抓取中国联通电子发票相关推荐

最新文章

热门文章