爬虫行政区号并获取电话区号和邮编

1、mysql建表语句

CREATE TABLE `his_sys_plat_region` (`Id` bigint(20) NOT NULL AUTO_INCREMENT,`RegionId` bigint(20) NOT NULL DEFAULT '0' COMMENT '地区ID',`ParentRegionId` bigint(20) NOT NULL DEFAULT '0' COMMENT '上级地区ID',`RegionName` varchar(50) DEFAULT NULL COMMENT '地区名称',`RegionNameEN` varchar(50) DEFAULT NULL COMMENT '英文地区名称',`Depth` int(11) NOT NULL DEFAULT '0' COMMENT '地区层级',`IdPath` varchar(255) DEFAULT NULL COMMENT '地区ID路径',`NamePath` varchar(255) DEFAULT NULL COMMENT '地区名称路径',`PostCode` varchar(20) DEFAULT NULL COMMENT '邮政编码',`TelephoneCode` varchar(20) DEFAULT NULL COMMENT '电话区号',`OrderNum` int(11) NOT NULL DEFAULT '0' COMMENT '排序号',`Lat` varchar(60) DEFAULT NULL COMMENT '地区中心经度',`Lng` varchar(60) DEFAULT NULL COMMENT '地区中心纬度',`Active` tinyint(4) NOT NULL DEFAULT '1' COMMENT '是否有效',`QueryStr` varchar(8000) DEFAULT NULL COMMENT '查询字符串',`CreateTime` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',`UpdateTime` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',PRIMARY KEY (`Id`),KEY `idx_his_sys_plat_region_rid` (`RegionId`),KEY `idx_his_sys_plat_region_pid` (`ParentRegionId`),KEY `idx_his_sys_plat_region_idp` (`IdPath`)
) ENGINE=InnoDB AUTO_INCREMENT=3194 DEFAULT CHARSET=utf8mb4;

2、对应的实体类

import java.util.Date;
import java.util.Map;import yb.common.util.InitListMap;
import yb.core.tablemodel.annotation.Column;
import yb.core.tablemodel.annotation.Table;
import yz.his.model.BaseModel;/*** 数据库表his_sys_plat_region的model类* * @author 系统自动生成* @create 2019-12-20*/
@Table(name = "his_sys_plat_region")
public class HisSysPlatRegionModel extends BaseModel {protected transient Map<String, String> fieldDictTypes = InitListMap.toHashMap(new String[] {});@Column(name = "Id", primaryKey = true, insertable = false)private long Id;@Column(name = "RegionId")private long RegionId = 0L; // 地区ID@Column(name = "ParentRegionId")private long ParentRegionId = 0L; // 上级地区ID@Column(name = "RegionName")private String RegionName; // 地区名称@Column(name = "RegionNameEN")private String RegionNameEN; // 英文地区名称@Column(name = "Depth")private int Depth = 0; // 地区层级@Column(name = "IdPath")private String IdPath; // 地区ID路径@Column(name = "NamePath")private String NamePath; // 地区名称路径@Column(name = "PostCode")private String PostCode; // 邮政编码@Column(name = "TelephoneCode")private String TelephoneCode; // 电话区号@Column(name = "OrderNum")private int OrderNum = 0; // 排序号@Column(name = "Lat")private String Lat; // 地区中心经度@Column(name = "Lng")private String Lng; // 地区中心纬度@Column(name = "Active")private int Active = 1; // 是否有效@Column(name = "QueryStr")private String QueryStr; // 查询字符串@Column(name = "CreateTime", insertable = false, updateable = false)private Date CreateTime; // 创建时间@Column(name = "UpdateTime", insertable = false, updateable = false)private Date UpdateTime; // 更新时间/*** Id*/public long getId() {return Id;}/*** Id*/public void setId(long Id) {this.Id = Id;}/*** 地区ID<br/>* default: 0*/public long getRegionId() {return RegionId;}/*** 地区ID<br/>* default: 0*/public void setRegionId(long RegionId) {this.RegionId = RegionId;}/*** 上级地区ID<br/>* default: 0*/public long getParentRegionId() {return ParentRegionId;}/*** 上级地区ID<br/>* default: 0*/public void setParentRegionId(long ParentRegionId) {this.ParentRegionId = ParentRegionId;}/*** 地区名称*/public String getRegionName() {return RegionName;}/*** 地区名称*/public void setRegionName(String RegionName) {this.RegionName = RegionName;}/*** 英文地区名称*/public String getRegionNameEN() {return RegionNameEN;}/*** 英文地区名称*/public void setRegionNameEN(String RegionNameEN) {this.RegionNameEN = RegionNameEN;}/*** 地区层级<br/>* default: 0*/public int getDepth() {return Depth;}/*** 地区层级<br/>* default: 0*/public void setDepth(int Depth) {this.Depth = Depth;}/*** 地区ID路径*/public String getIdPath() {return IdPath;}/*** 地区ID路径*/public void setIdPath(String IdPath) {this.IdPath = IdPath;}/*** 地区名称路径*/public String getNamePath() {return NamePath;}/*** 地区名称路径*/public void setNamePath(String NamePath) {this.NamePath = NamePath;}/*** 邮政编码*/public String getPostCode() {return PostCode;}/*** 邮政编码*/public void setPostCode(String PostCode) {this.PostCode = PostCode;}/*** 电话区号*/public String getTelephoneCode() {return TelephoneCode;}/*** 电话区号*/public void setTelephoneCode(String TelephoneCode) {this.TelephoneCode = TelephoneCode;}/*** 排序号<br/>* default: 0*/public int getOrderNum() {return OrderNum;}/*** 排序号<br/>* default: 0*/public void setOrderNum(int OrderNum) {this.OrderNum = OrderNum;}/*** 地区中心经度*/public String getLat() {return Lat;}/*** 地区中心经度*/public void setLat(String Lat) {this.Lat = Lat;}/*** 地区中心纬度*/public String getLng() {return Lng;}/*** 地区中心纬度*/public void setLng(String Lng) {this.Lng = Lng;}/*** 是否有效<br/>* default: 1*/public int getActive() {return Active;}/*** 是否有效<br/>* default: 1*/public void setActive(int Active) {this.Active = Active;}/*** 查询字符串*/public String getQueryStr() {return QueryStr;}/*** 查询字符串*/public void setQueryStr(String QueryStr) {this.QueryStr = QueryStr;}/*** 创建时间<br/>* default: CURRENT_TIMESTAMP*/public Date getCreateTime() {return CreateTime;}/*** 创建时间<br/>* default: CURRENT_TIMESTAMP*/public void setCreateTime(Date CreateTime) {this.CreateTime = CreateTime;}/*** 更新时间<br/>* default: CURRENT_TIMESTAMP*/public Date getUpdateTime() {return UpdateTime;}/*** 更新时间<br/>* default: CURRENT_TIMESTAMP*/public void setUpdateTime(Date UpdateTime) {this.UpdateTime = UpdateTime;}@Overridepublic Map<String, String> getFieldDictTypes() {return fieldDictTypes;}
}

3、同步代码

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;import yb.common.util.Http;
import yb.common.util.ObjectTransfer;
import yb.common.util.Tester;
import yb.core.bundle.annotation.YB;
import yb.core.business.YBService;
import yb.core.dao.IDao;
import yb.core.loader.BundleLoader;
import yb.module.pinyin.PinyinFormat;
import yb.module.pinyin.PinyinHelper;
import yz.his.common.util.LocationPoint;
import yz.his.common.util.NameKeywordsUtil;
import yz.his.model.sys.HisSysPlatRegionModel;@YB
public class SyncRegionServer extends YBService {private static final String PROV = "province";private static final String CITY = "city";private static final String REGION = "region";private String getData(String url) throws Exception {if (url == null || url.isEmpty())return "";String result = "";Http http = new Http();result = new String(http.get(url), "utf-8");return result;}private List<HisSysPlatRegionModel> getDataStructure(String str) throws Exception {// 想要获取字段部分的分割模式final Pattern provMatcher = Pattern.compile("<td height=19 class=xl658256 style='height:14.25pt'></td>\\s+" + "<td class=xl708256>(.*)</td>\\s+"+ "<td class=xl708256>(.*)</td>\\s+" + "<td class=xl658256></td>\\s+"+ "<td class=xl698256></td>\\s+" + "<td class=xl698256></td>\\s+"+ "<td class=xl698256></td>\\s+" + "<td class=xl638256></td>\\s+" + "<td class=xl638256></td>",Pattern.DOTALL);/** final Pattern cityMatcher = Pattern.compile(* "<td height=19 class=xl658256 style='height:14.25pt'></td>\\s+" +* "<td class=xl708256>(.*)</td>\\s+" +* "<td class=xl708256><span style='mso-spacerun:yes'> </span>(.*)</td>\\s+"* + "<td class=xl658256></td>\\s+" + "<td class=xl698256></td>\\s+" +* "<td class=xl698256></td>\\s+" + "<td class=xl698256></td>\\s+" +* "<td class=xl638256></td>\\s+" + "<td class=xl638256></td>",* Pattern.DOTALL);*/final Pattern regionMatcher = Pattern.compile("<td height=19 class=xl658256 style='height:14.25pt'></td>\\s+"+ "<td class=xl718256>(.*)</td>\\s+"+ "<td class=xl718256><span style='mso-spacerun:yes'>   </span>(.*)</td>\\s+"+ "<td class=xl658256></td>\\s+" + "<td class=xl698256></td>\\s+" + "<td class=xl698256></td>\\s+"+ "<td class=xl698256></td>\\s+" + "<td class=xl638256></td>\\s+" + "<td class=xl638256></td>",Pattern.DOTALL);final Pattern specialMatcher = Pattern.compile("<td height=19 class=xl698256 style='height:14.25pt'></td>\\s+"+ "<td class=xl708256>(.*)</td>\\s+" + "<td class=xl708256>(.*)</td>\\s+"+ "<td class=xl698256></td>\\s+" + "<td class=xl698256></td>\\s+" + "<td class=xl698256></td>\\s+"+ "<td class=xl698256></td>\\s+" + "<td class=xl638256></td>", Pattern.DOTALL);// 想要获取的整段数据的分割模式String[] info = str.split("</tr>");Map<Long, HisSysPlatRegionModel> entityMap = new HashMap<>();List<HisSysPlatRegionModel> list = new ArrayList<HisSysPlatRegionModel>();HisSysPlatRegionModel china = new HisSysPlatRegionModel();china.setRegionId(86L);china.setRegionName("中国");china.setDepth(1);china.setIdPath("|86|");china.setNamePath("中国");list.add(china);// 中国Matcher pm = null;Matcher cm = null;Matcher rm = null;Matcher sm = null;Matcher matcher = null;String belong = null;for (String s : info) {pm = provMatcher.matcher(s);rm = regionMatcher.matcher(s);sm = specialMatcher.matcher(s);if (pm.find()) {matcher = pm;if (!pm.group(2).contains("<")) {belong = PROV;} else {belong = CITY;}} else if (rm.find()) {belong = REGION;matcher = rm;} else if (sm.find()) {belong = PROV;matcher = sm;} else {continue;}HisSysPlatRegionModel entity = getEntity(matcher, belong, entityMap);list.add(entity);}return list;}private HisSysPlatRegionModel getEntity(Matcher matcher, String belong, Map<Long, HisSysPlatRegionModel> entityMap)throws Exception {HisSysPlatRegionModel entity = new HisSysPlatRegionModel();String regionId = matcher.group(1);String regionName = matcher.group(2);entity.setRegionId(ObjectTransfer.longValue(regionId));Long key = null;HisSysPlatRegionModel value = null;LocationPoint location = null;switch (belong) {case PROV:entity.setParentRegionId(86L);entity.setDepth(2);entity.setIdPath("|86|" + regionId + "|");entity.setNamePath("中国|" + regionName);if (regionName.endsWith("市")) {List<String> telephoneCodeAndPostCode = getTelephoneCodeAndPostCode(regionName);System.out.println(ObjectTransfer.print(telephoneCodeAndPostCode));if (telephoneCodeAndPostCode != null && telephoneCodeAndPostCode.size() == 2) {entity.setTelephoneCode(telephoneCodeAndPostCode.get(0));entity.setPostCode(telephoneCodeAndPostCode.get(1));}}break;case CITY:regionName = regionName.replace("<span style='mso-spacerun:yes'> </span>", "");key = ObjectTransfer.longValue(regionId.substring(0, 2).concat("0000"));if ((value = entityMap.get(key)) != null) {entity.setParentRegionId(key);entity.setIdPath(value.getIdPath() + regionId + "|");entity.setNamePath(value.getNamePath() + "|" + regionName);entity.setDepth(3);List<String> telephoneCodeAndPostCode = getTelephoneCodeAndPostCode(regionName);System.out.println(ObjectTransfer.print(telephoneCodeAndPostCode));if (telephoneCodeAndPostCode != null && telephoneCodeAndPostCode.size() == 2) {entity.setTelephoneCode(telephoneCodeAndPostCode.get(0));entity.setPostCode(telephoneCodeAndPostCode.get(1));}}// location =// BaiduUtil.getLocationByAddress(entity.getNamePath().replace("|",// ""));// if (location != null) {// entity.setLng(String.valueOf(location.getLongtitude()));// entity.setLat(String.valueOf(location.getLatitude()));// }break;case REGION:key = ObjectTransfer.longValue(regionId.substring(0, 4).concat("00"));if ((value = entityMap.get(key)) != null) {entity.setDepth(4);} else {key = ObjectTransfer.longValue(regionId.substring(0, 2).concat("0000"));value = entityMap.get(key);entity.setDepth(3);}entity.setParentRegionId(key);entity.setIdPath(value.getIdPath() + regionId + "|");entity.setNamePath(value.getNamePath() + "|" + regionName);// location =// BaiduUtil.getLocationByAddress(entity.getNamePath().replace("|",// ""));// if (location != null) {// entity.setLng(String.valueOf(location.getLongtitude()));// entity.setLat(String.valueOf(location.getLatitude()));// }break;default:break;}entity.setRegionName(regionName);String queryStr = getQueryStr(regionName);entity.setQueryStr(queryStr);entityMap.put(ObjectTransfer.longValue(regionId), entity);return entity;}private String getQueryStr(String word) {if (word == null || word.isEmpty()) {return "";}StringBuffer sb = new StringBuffer(word);Map<String, String> map = NameKeywordsUtil.pinyinOfName(word);for (Entry<String, String> entry : map.entrySet()) {if (entry == null || entry.getKey() == null)continue;sb.append("|" + entry.getKey());}return sb.toString();}private void start() throws Exception {IDao dao = dao("his");String url = "http://www.mca.gov.cn/article/sj/xzqh/2019/2019/201911250933.html";String data = getData(url);List<HisSysPlatRegionModel> resultList = getDataStructure(data);dao.insertList(resultList);}public List<String> getTelephoneCodeAndPostCode(String area) throws Exception {if (area == null || area.isEmpty()) {return null;}String search = "";if (area.endsWith("市"))search = area.replace("市", "");if (area.endsWith("盟"))search = area.replace("盟", "");if (area.endsWith("地区"))search = area.replace("地区", "");if (area.endsWith("州"))search = area.substring(0, 2);String convertToPinyinString = PinyinHelper.convertToPinyinString(search, "", PinyinFormat.WITHOUT_TONE);String url = "http://quhao.tianqi.com/" + convertToPinyinString + "/";Pattern tcPattern = Pattern.compile("<td><a title=\"(.*)是哪里的区号\" target=\"_blank\" href=\"/(.*)/\">(.*)</a></td>");Pattern pcPattern = Pattern.compile("<td><a title=\"(.*)是哪里的邮编\" target=\"_blank\" href=\"http://youbian.tianqi.com/"+ convertToPinyinString + "/\">(.*)</a>");Http http = new Http();byte[] bs = http.get(url);String result = new String(bs, "utf-8");Matcher tcMatcher = tcPattern.matcher(result);Matcher pcMatcher = pcPattern.matcher(result);List<String> list = new ArrayList<>();if (tcMatcher.find()) {list.add(tcMatcher.group(1));} else {String url2 = "https://www.chahaoba.com/%E4%B8%AD%E5%9B%BD%E7%94%B5%E8%AF%9D%E5%8C%BA%E5%8F%B7?search="+ area;byte[] bs2 = http.get(url2);String tc = new String(bs2, "utf-8");Pattern pattern = Pattern.compile("<ul><li> 国内拨打：<a href=\"/21\" title=\"21\">(.*)</a></li>");Matcher matcher = pattern.matcher(tc);if (matcher.find()) {list.add(matcher.group(1));}list.add("");return list;}if (pcMatcher.find()) {list.add(pcMatcher.group(1));}return list;}/** main test* * */public static void main(String[] args) {try {Tester.init();BundleLoader loader = Tester.initContext(3000, 3000);SyncRegionServer server = loader.getBean(SyncRegionServer.class);long startTime = System.currentTimeMillis();server.start();System.out.println("共计用时：" + (System.currentTimeMillis() - startTime) / 1000);} catch (Exception e) {e.printStackTrace();} finally {Tester.close();}}
}

4、实际数据库效果

爬虫行政区号并获取电话区号和邮编相关推荐

10分钟采集凡客最新的省、市、区、邮政编码和电话区号（附源码）
最近的开发的项目需要用到省.市.区数据,因为要开发的项目也是电子商务网站,在参考凡客的用户体验时,发现它连深圳最新分离出来的光明新区都有了,拍拍网都没有更新数据,看来凡客在数据更新方面还是挺负责的,所 ...
电话区号信息API接口，免费好用
1.前言电话区号信息查询接口,能实现获取世界电话区号列表.这个接口的特点是数据会经常更新. 查看接口完整信息:https://www.idmayi.com/doc/detail?id=6 2.接口明 ...
iOS判断国内固定电话区号
最近在项目中需要对用户填写的固定电话做处理希望要的格式为例如 010-12345678 样式的. 但很不巧的是我在iOS客户端可以严格约束用户输入指定格式的固定电话(使用两个文本框,一个用来接收区号, ...
Select下拉框json获取国家区号
应客户需求,填写电话区号时,要求下拉选择国家来获取该国家的电话区号. 代码如下: 国家区码: country.json [{"cnname": "中国大陆",& ...
中国城市电话区号对照表中国移动短信中心号查询及命名规则
中国移动短信中心号查询及命名规则开头:+861380 中间:城市电话区号,不足4位以0结尾结尾:500 列如:+861380xxxx500 总共15位数字中国城市电话区号对照表连接如下: htt ...
空间换时间--编程小绝招解决电话区号识别算法问题
空间换时间问题通常都是一个不大的问题里的一个不小的解决方案.首先要理解代码中函数的一个大致的代价,在输入比较固定,计算输出又比较费劲的时候,常常就是空间换时间发挥作用的时候了.通过几个简单的例子您就会 ...
国际电话区号--各国或地区电话国际区号对照表
具体参考:国际电话区号--各国或地区电话国际区号对照表
jQuery国际电话区号选择插件intlTelInput.js
jQuery国际电话区号选择插件intlTelInput.js URL: http://www.jq22.com/jquery-info12917 jQuery世界地区三级联动 URL: http ...
中国行政区编码_邮政编码_区号编码
原文:中国行政区编码_邮政编码_区号编码源代码下载地址:http://www.zuidaima.com/share/1550463699946496.htm 中国行政区编码_邮政编码_区号编码SQL ...

爬虫行政区号并获取电话区号和邮编

爬虫行政区号并获取电话区号和邮编

爬虫行政区号并获取电话区号和邮编

1、mysql建表语句

2、对应的实体类

3、同步代码

4、实际数据库效果

爬虫行政区号并获取电话区号和邮编相关推荐

最新文章

热门文章