python爬取四川建设行业数据共享平台

1.直接上代码

本人第一次写博客，文笔不好，有问题留言评论吧！

import csv
import requests
import time
import tools
from lxml import etree
url = 'http://jst.sc.gov.cn/xxgx/Enterprise/eLWQYList.aspx'
def getres(VIEWSTATE,EVENTVALIDATION,page):session=requests.session()header="""Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8Accept-Encoding: gzip, deflateAccept-Language: zh-CN,zh;q=0.9Cache-Control: no-cacheConnection: keep-aliveHost: jst.sc.gov.cnPragma: no-cacheUpgrade-Insecure-Requests: 1User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"""data="""__VIEWSTATEGENERATOR: 763777C1
__EVENTTARGET: ctl00$MainContent$gvBiddingResultPager
mc:
xydm: """datas=tools.make_header(data)datas['__VIEWSTATE'] = VIEWSTATEdatas['__EVENTVALIDATION'] = EVENTVALIDATIONdatas['__EVENTARGUMENT'] = pageheaders=tools.make_header(header)res = session.post(url,headers=headers,data=datas)html=etree.HTML(res.text)item={}item['gsname'] = html.xpath("//table[@class='table  table-striped table-hover table-search-list']/tbody/tr/td[1]/text()")item['address'] = html.xpath("//table[@class='table  table-striped table-hover table-search-list']/tbody/tr/td[2]/text()")item['name'] = html.xpath("//table[@class='table  table-striped table-hover table-search-list']/tbody/tr/td[3]/text()")item['phone'] = html.xpath("//table[@class='table  table-striped table-hover table-search-list']/tbody/tr/td[4]/text()")item['VIEWSTATE'] = html.xpath("//input[@name='__VIEWSTATE']/@value")[0]item['EVENTVALIDATION'] =html.xpath("//input[@name='__EVENTVALIDATION']/@value")[0]item['nowpage']= html.xpath('//td[@class="paginator-custom-data"]/u[1]/text()')[0].split('/')[0]item['data'] = list(zip(item['gsname'], item['address'], item['name'], item['phone']))return itemdef save(item):with open('sichuan.csv','a',encoding='gbk',newline='') as f:csvwriter=csv.writer(f)for i in item['data']:csvwriter.writerow(i)print('写入完成')if __name__=='__main__':a= 'uWhrfmebaKkUMlffMLJINcIGP+xxigPFuw+qi8SvI7HO0pWSTZawQGAqnOmTL5HdWKBQ1Gl+LLD/sVlTW6shPRAhUS/REK9rGuoSaPLeM/C7mKApWRCevAIzV4S6Q8HItzGrbDgEgMs9fv9OGzKhW0J/suq/eOuGd0wEgWb7lHP2EXULq6rqct3K/vwndeXEl2xoI8Iwvs23UZOyUCV1zBKhLIBTOBNJpzg/HEAFgVczbO/KVjXlUaEs9co8H0kbYuCom566LeyYYZMvI8gXhLdeQOdC0GSBdvDmUgfBcDREI1wHjGaqPltxX2EHFp+9o2lQ1+Dt5YKGq2Yy5D3GfVx+/rVslT9RRhj6kXVoSDjA1x5VGxEnobnR5TzCJa5I9KaaCNXRM8MuhEdzpxAFUcjIOf79jMWxPg6w/3j8tYkKGcEpIocAJ8scyReFydqGGlFeWwbmRqa7URWGLpDOS6ltrYxMDDOczGf5FK40dxP95wtxQ7wxDQuhLCy+QXLixJwZ2LPdZCKk5Cqc/9OVlZo7eCN8b3KdlchqLiGvfRhgQdRZ+KFjv6sc0GInr6JkB4oRuug+20IftkWXNO4cV5EfQGdPbcd57RBcezs0Lc0rbKGoFC+JiCPaNO07GKglOOipUbLLVb0aN2DiAzn4UwPGHrCzMJZUkSvn5iAZX9oOmUoqJUmz+jhnIJUqaH9a5u0e/oq+G1KgENPjCfMNIa/mLHMeTsMQxS8zKt6NMXQZmzQ/gMXii8vQqGtUes107KOpaaLZ3fEQRKE7tN0SDjawY1dG9M5tYjsHSNTLM5wSJI2RMxqN+Iq2CcIiVLA9ux0we/RZmBnZPhfBIHnQ2/axAzn/d2eaL9F9tkW857/wrNxM6K2vpptsbeEg25pIjlQ/3NmWosQQEAk0y+9mtD8FwalALGPxkMw7CNaM7AK1LCnrHKNgQmoGrRpmhHZZVo97gye+QZJ7046s6G7pEgcTqdxPW2CIN1wzwrQLpaojs3QWTc4vlyjDu4AFeqjx/hfa8uxw5Lu0SCplkcuoUKDVOLgN7p6yBjIzycOJSCBAL6aRS5ArCABRDy1G8IG+VP7MnUC8NYqKpxFOvjydstzKR8HN7yBF+amMekiDwvLa4P1Li2kTj1LlJZ4qv+8klKreD28nOLeBQVyrMkY+23nQeLzBtMC5fOJvk9bv5Wa2q9M0gTo+EIgBp7xz8vgIx9ehkx6F76Z0yOx79+h7L+qMvJZEMhas+kUNplIiX5u0HaSeTupmhd30THLFjXOvI1KOmk+OIwMS4vyfeCQGcgeQ5JplUVdHBHFaRUsjSG/79XhNtqb+AFyNG1hUS4pKnnjEdfmUeoTZfl/xK/lo3x+q8uSOx7jEg3L93DZx0jeLgEM2Bm8GTIYtwt9415RtiY2N9v7yJMTtIfTG3wW8aqIZOwHyNs/BjVEYrllq90fv2TvjIBA+SfDDMOYkWqs0BZxCU1Ra1efy++fdEIIc6jw8CkUdO8xXo7tdWHbKRt7NdK4rd9htQRAK9Ui/ifEStc3DIXOGXtJLw8RmOqskj7dAkR6frBhDrNNr7NnRDgXzTbVsjOP/fo/A8r6d3FjFBMsVroSvOVrhmWPusbTMRwWSd0QYfpZoJxE4buSGEGXyi3wHcqARubHf0M/qMtl5rdbUWkkuYesn+e7Q1blw9+cxNYSxAFgA9xmcOWlWBWvG47/OPXLJdfpcZ+NLJJpnQvbfQUAzQR6Rz1Q8qVPbhXmgv+6UweJ2tNUDdtBb4NbaiFzSw5l602Ohaf7JzgYcncKuf9ETuJypwXMnpq1LvNFil8Y54Nj39tXYrVGVD8NfCLlcS4jawNsO9TsIVtHiEYhEOME7fSAkBAnfRVmZJxrHP7sYG9rmP8AwOg9UEW7MbtyEswBIuobEisZ8g/LbxMLsbZi+ihpWyV4it1qDNz45jc7ZVhZ+yH8X8RUe81Msly6PSCyLjfj8Ljn2vDacccvWSnZc74qGFwIrYB53p+8BZAOWsgpr9hIDvn+pwHX92818PQBXI1G7U+VpjLYrOCnr/JAOVBW2hdlvmzyxNmlmPcWxNTfHNdIKFTUCaLv6qP344V0A5MAPLUmr7Xi57T3FM43JgXdaFCArUrso9GbzEJiLa88qpmET2jQfXNriHWWcAKJMeRpFDWC8mgbmS/2CC2FVdg1AoB09lUhMhXhxBB97N/EsHKVYeFpRER7DaCXW/3DY00HnMZbY04eYGAIok0o6JkWZXJaT8LvT3ft84eD2m+KaA9cTpPZUuwFSvrmjX3P5SfDPsO6mGAKsrCEiS4Y9KfW3MCM4hW/gVW/tazBSb6R5vq271OtMexnfHQFbRCAWyjoMpdig7VeaknvB5Eq80w1wS7u62d/PVwZo6R3i1JXF5+8h321+DHjy48icXJKW8FAXpfyE5AUGWPFFsZudRmo4/f1MIfy/jH+dSHvVEQBGJ3FvCz7W7hbUQrGpJSFPZyvxz2M9QQ2/r0vn+fWin3zUYGP0onQx6luWTW7GSwZuCvsbYALemcG+D+8kmU/M0u7n3rOeK8HluaMrY1G1heBKfB1TrPH1BLRpwvuIEzqXSH7o1oe+MYpvvNyPZnANIZirBXuhT//UPuwqF5JaPJ+2MmMe+XgKK+Jo+k6nKbt6SoRWLatdSAIH/gv9JN0NQGMDqdPw+dXyYX+hcsu0bhFfkMILA8yJCD0ppZoYGs7ad+Vx+xftYdHiACNxd27WVGWJ0haD+x1y0qiTZ1z0uTaD/6fygsaOveEivXs='b='+KQiT6UadREnecRVPJKD4udT2L2rpITB/YV0GOSWU8N1M+IPWe/9IeKqxhjyWuEP8u7EWFZ/4G3yJS9Q5yThGFZy9pQjwHXPACFUG495ZMaja4w63+8QlNhkiHSnDglnUe/c7/2REyA'first=getres(a,b,7)a = first['VIEWSTATE']b = first['EVENTVALIDATION']page=first['nowpage']save(first)# print(first)# exit()for i in range(1928):res = getres(a,b,str(int(page)+1))save(res)a = res['VIEWSTATE']b = res['EVENTVALIDATION']page = res['nowpage']print('爬取一页，当前页数'+page)time.sleep(1)

tools.py

# codeing=utf-8
"""
Author:song
"""
""" 参数为浏览器直接复制的header字符串，返回处理好的headers字典 """
def make_header(header_str=''):headers = header_str.split('\n')# return headersheaders_dict = {i.split(': ')[0].strip(): i.split(': ')[1].strip() for i in headers}return headers_dict""" 参数为浏览器直接复制的cookie字符串，返回处理好的cookies字典 """
def make_cookie(cookie=''):cookies= {i.split("=")[0]:i.split("=")[1] for i in cookie.split(";")}return cookies

PS：仅供学习使用，请勿用于非法用途，出现任何问题与本人无关

python爬取四川建设行业数据共享平台相关推荐

用python爬取网贷之家p2p平台数据
网贷之家中的p2p平台数据比较容易获取,重要的就是如何分析网页的源代码然后从里面提取自己需要的信息,也不需要用户登录,该网站的爬虫比较简单,主要用了urllib包来获取网页信息,用BeautifulS ...
python爬取客流数据_爬取首都机场客流量数据，从GIS的角度尝试分析建设大兴机场的必要性...
Hello, 大家好!我是James. 上一节我们整理好了北京的行政区和街道办矢量数据,估计跃跃欲试的小伙伴太热情,爬取的网站被玩坏了几天,学习可以,但是不要过多尝试了哈然后在文章的最后,我们惊奇的 ...
python大数据平台_基于腾讯位置大数据平台的全球移动定位数据Python爬取与清洗...
前不久投稿了一篇论文是以腾讯位置大数据为基础进行人口空间化研究的,但是还未见刊,见刊后会给大家分享下具体的研究方法. 首先打开腾讯位置大数据星云图链接:https://xingyun.map.qq.c ...
【爬虫】Python爬取电商平台评论完整代码
利用Ajax爬取淘宝评论,这里完整的补充一下,包括数据存储. 对于Ajax参数的分析,Python爬取平台评论,这篇文章分析过了这里不再重复了. 主要是完善一下代码. import time impo ...
爬取电商平台数据，python爬取某维商品数据
本次内容: 爬取电商平台数据,python爬取某维商品数据课程亮点动态数据抓包演示 json数据解析 requests模块的使用保存csv 环境介绍 python 3.8 [最好用和老师一样的版 ...
学python能赚什么外卖-python爬取外卖
广告关闭腾讯云双11爆品提前享,精选热门产品助力上云,云服务器首年88元起,买的越多返的越多,最高满返5000元! time.sleep(1)d:pythonvenvscriptspython.ex ...
你的城市撒币了吗？Python 爬取分析全国消费券发放数据
作者 | 刘早起来源 | 早起Python(ID: zaoqi-python) 近期,全国多地以各种形式投放消费券.消费补贴来鼓励消费,部分城市在首期消费券的基础上,连续追加发放多期消费券.你在的城 ...
如何用 Python 爬取网页制作电子书
本文来自作者孙亖在 GitChat 上分享「如何用 Python 爬取网页制作电子书」,「阅读原文」查看交流实录. 「文末高能」编辑 | 哈比 0 前言有人爬取数据分析黄金周旅游景点,有人爬 ...
手把手教你用python爬取人人贷网站借款人信息
P2P是近年来很热的一个行业,由于这个行业在国内兴起才不久,国内的很多学者对这个行业都兴趣盎然,在大学研究互联网金融的学者更是有一大群.小编是学金融出身,深知数据在做学术研究的重要性,之前有不少学互联 ...

python爬取四川建设行业数据共享平台

1.直接上代码

python爬取四川建设行业数据共享平台相关推荐

最新文章

热门文章