纯真IP地址数据库qqwry.dat解析

ip地址数据库，在现在互联网时代非常有用，比如大型网站的用户安全保护系统，就常常会根据ip反查的信息，甄别账号的一些不安全登录行为，比如跨区域登录问题等。ip其实关联了一些有信息，比如区域，所在运营商，一些收录全的，甚至包括具体经纬度，像百度的IP定位api就比较全。下面来介绍一下“ 纯真IP地址数据库qqwry”的格式以及解析

以下是“ 纯真IP地址数据库qqwry”官网对其的介绍。

纯真版IP地址数据库是当前网络上最权威、地址最精确、IP记录以及网吧数据最多的IP地址数据库。收集了包括中国电信、中国移动、中国联通、铁通、长城宽带等各 ISP 的最新准确 IP 地址数据。通过大家的共同努力打造一个没有未知数据，没有错误数据的QQ IP。IP数据库每5天更新一次，请大家定期更新最新的IP数据库！

格式

+———-+
| 文件头 | (8字节)
+———-+
| 记录区 | (不定长)
+———-+
| 索引区 | (大小由文件头决定)
+———-+

使用java语言解析的两种思路：

使用内存映射文件方式读取，使用java的MappedByteBuffer 将原数据文件映射到MappedByteBuffer对象中，然后通过MappedByteBuffer 提供的字节读取方式实现ip的查找。搜索是在索引区使用二分法
使用byte数组读取，及将二进制的数据库信息全都按顺序读入到一个数组中，由于数据是有格式的，我们便可计算根据索引区和记录区在数组中的位置，当查询ip时，从数组中的索引区开始通过二分查找方式找到IP地址对应的国家和区域的位置，然后从数组中取出地区信息。

热升级思路：

使用一个可调度的单线程的线程池，线程定时检测qqwry.dat文件是否修改，若修改则重新将数据重新载入，载入过程可使用可重入锁ReentrantLock来锁住资源，避免在更新的过程中脏查询

两种解析方式的实现源码如下：
方式一（MappedByteBuffer ）：

package com.difeng.qqwry1;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
/*** @Description:ip定位查找工具(使用内存映射文件方式读取，线程安全)* @author:difeng* @date:2016年12月11日*/
public class IPLocation {private static final int IP_RECORD_LENGTH = 7;private static final byte REDIRECT_MODE_1 = 0x01;private static final byte REDIRECT_MODE_2 = 0x02;private MappedByteBuffer mbbFile;private static Long lastModifyTime = 0L;public static boolean enableFileWatch = false;private static ReentrantLock lock = new ReentrantLock();private File qqwryFile;private long firstIndexOffset;private long lastIndexOffset;private long totalIndexCount;public IPLocation(String filePath) throws Exception {this.qqwryFile = new File(filePath);load();if (enableFileWatch) {watch();}}private void watch(){Executors.newScheduledThreadPool(1).scheduleAtFixedRate(new Runnable() {@Overridepublic void run() {long time = qqwryFile.lastModified();if (time > lastModifyTime) {lastModifyTime = time;try {load();} catch (Exception e) {e.printStackTrace();}}}}, 1000L, 30000L, TimeUnit.MILLISECONDS);}public long read4ByteAsLong(long pos) {mbbFile.position((int)pos);return 0xFFFFFFFFL & mbbFile.getInt();}public long read3ByteAsLong(long pos){mbbFile.position((int)pos);return 0xFFFFFFL & mbbFile.getInt();}@SuppressWarnings("resource")private void load() throws Exception {lastModifyTime = qqwryFile.lastModified();lock.lock();try {mbbFile =  new RandomAccessFile(qqwryFile, "r").getChannel().map(FileChannel.MapMode.READ_ONLY, 0, qqwryFile.length());mbbFile.order(ByteOrder.LITTLE_ENDIAN);firstIndexOffset = read4ByteAsLong(0);lastIndexOffset = read4ByteAsLong(4);totalIndexCount = (lastIndexOffset - firstIndexOffset) / IP_RECORD_LENGTH + 1;} finally {lock.unlock();}}/*** @Description:将“.”号分隔的字符串转换为long类型的数字，字节序例如:*   ip:182.92.240.48  16进制表示(B6.5C.F0.30)*   转换后ip的16进制表示:0xB65CF030* @param ipStr* @return:long*/private static long inet_pton(String ipStr) {if(ipStr == null){throw new NullPointerException("ip不能为空");}String [] arr = ipStr.split("\\.");long ip = (Long.parseLong(arr[0])  & 0xFFL) << 24 & 0xFF000000L;ip |=  (Long.parseLong(arr[1])  & 0xFFL) << 16 & 0xFF0000L;ip |=  (Long.parseLong(arr[2])  & 0xFFL) << 8 & 0xFF00L;ip |=  (Long.parseLong(arr[3])  & 0xFFL);return ip;}private long search(long ip) {long low = 0;long high = totalIndexCount;long mid = 0;while(low <= high) {mid = (low + high) >>> 1 ;long indexIP = read4ByteAsLong(firstIndexOffset + (mid - 1) * IP_RECORD_LENGTH);long nextIndexIP =  read4ByteAsLong(firstIndexOffset + mid * IP_RECORD_LENGTH);if(indexIP <= ip && ip < nextIndexIP) {return read3ByteAsLong(firstIndexOffset + (mid - 1) * IP_RECORD_LENGTH + 4);} else {if(ip > indexIP) {low = mid + 1;} else if(ip < indexIP) {high = mid - 1;}}}return -1;}private Location readIPLocation(long offset) {try {mbbFile.position((int)offset + 4);Location loc = new Location();byte redirectMode = mbbFile.get();if (redirectMode == REDIRECT_MODE_1) {long countryOffset = read3ByteAsLong((int)offset + 5);mbbFile.position((int)countryOffset);redirectMode = mbbFile.get();if (redirectMode == REDIRECT_MODE_2) {loc.country = readString(read3ByteAsLong(countryOffset + 1));mbbFile.position((int)countryOffset + 4);} else {loc.country = readString(countryOffset);}loc.area = readArea(mbbFile.position());} else if (redirectMode == REDIRECT_MODE_2) {loc.country = readString(read3ByteAsLong((int)offset + 5));loc.area = readArea((int)offset + 8);} else {loc.country = readString(mbbFile.position() - 1);loc.area = readArea(mbbFile.position());}return loc;} catch (Exception e) {return null;}}private String readArea(int offset) {mbbFile.position(offset);byte redirectMode = mbbFile.get();if (redirectMode == REDIRECT_MODE_1 || redirectMode == REDIRECT_MODE_2) {long areaOffset = read3ByteAsLong((int)offset + 1);if (areaOffset == 0){return "";} else {return readString(areaOffset);}} else {return readString(offset);}}private String readString(long offset) {try {mbbFile.position((int)offset);byte[] buf = new byte[128];int i;for (i = 0, buf[i] = mbbFile.get(); buf[i] != 0; buf[++i] = mbbFile.get());if (i != 0){return new String(buf, 0, i, "GBK");}} catch (IOException e) {e.printStackTrace();}return "";}public  Location fetchIPLocation(String ip) {lock.lock();try {long offset = search(inet_pton(ip));if(offset != -1){return readIPLocation(offset);}} finally {lock.unlock();}return null;}
}

方式二（数组方式）：

package com.difeng.qqwry2;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
/*** @Description:ip定位(使用byte数据方式读取)* @author:difeng* @date:2016年12月13日*/
public class IPLocation {private  byte[] data;private  long firstIndexOffset;private  long lastIndexOffset;private  long totalIndexCount;private static final byte REDIRECT_MODE_1 = 0x01;private static final byte REDIRECT_MODE_2 = 0x02;static   final long IP_RECORD_LENGTH = 7;private static ReentrantLock lock = new ReentrantLock();private static Long lastModifyTime = 0L;public static boolean enableFileWatch = false;private File qqwryFile;public IPLocation(String  filePath) throws Exception {this.qqwryFile = new File(filePath);load();if(enableFileWatch){watch();}}private void watch() {Executors.newScheduledThreadPool(1).scheduleAtFixedRate(new Runnable() {@Overridepublic void run() {long time = qqwryFile.lastModified();if (time > lastModifyTime) {lastModifyTime = time;try {load();System.out.println("reload");} catch (Exception e) {e.printStackTrace();}}}}, 1000L, 5000L, TimeUnit.MILLISECONDS);}private void load() throws Exception {lastModifyTime = qqwryFile.lastModified();ByteArrayOutputStream out = null;FileInputStream in = null;lock.lock();try {out = new ByteArrayOutputStream();byte[] b = new byte[1024];in = new FileInputStream(qqwryFile);while(in.read(b) != -1){out.write(b);}data = out.toByteArray();firstIndexOffset = read4ByteAsLong(0);lastIndexOffset = read4ByteAsLong(4);totalIndexCount = (lastIndexOffset - firstIndexOffset) / IP_RECORD_LENGTH + 1;in.close();out.close();} finally {try {if(out != null) {out.close();}if(in != null) {in.close();}} catch (IOException e) {e.printStackTrace();}lock.unlock();}}private long read4ByteAsLong(final int  offset) {long val = data[offset] & 0xFF;val |= (data[offset + 1] << 8L) & 0xFF00L;val |= (data[offset + 2] << 16L) & 0xFF0000L;val |= (data[offset + 3] << 24L) & 0xFF000000L;return val;}private long read3ByteAsLong(final int offset) {long val = data[offset] & 0xFF;val |= (data[offset + 1] << 8) & 0xFF00;val |= (data[offset + 2] << 16) & 0xFF0000;return val;}private long search(long ip) {long low = 0;long high = totalIndexCount;long mid = 0;while(low <= high){mid = (low + high) >>> 1 ;long indexIP = read4ByteAsLong((int)(firstIndexOffset + (mid - 1) * IP_RECORD_LENGTH));long indexIPNext = read4ByteAsLong((int)(firstIndexOffset + mid * IP_RECORD_LENGTH));if(indexIP <= ip && ip < indexIPNext) {return read3ByteAsLong((int)(firstIndexOffset + (mid - 1) * IP_RECORD_LENGTH + 4));} else {if(ip > indexIP) {low = mid + 1;} else if (ip < indexIP) {high = mid - 1;}}}return -1;}public Location fetchIPLocation(String ip) {long numericIp = inet_pton(ip);lock.lock();long offset = search(numericIp);try{if(offset != -1) {return readIPLocation((int)offset);}} finally {lock.unlock();}return null;}private Location readIPLocation(final int offset) {final Location loc = new Location();try {byte redirectMode = data[offset + 4];if (redirectMode == REDIRECT_MODE_1) {long countryOffset = read3ByteAsLong((int)offset + 5);redirectMode = data[(int)countryOffset];if (redirectMode == REDIRECT_MODE_2) {final QQwryString country = readString((int)read3ByteAsLong((int)countryOffset + 1));loc.country = country.string;countryOffset = countryOffset + 4;} else {final QQwryString country = readString((int)countryOffset);loc.country = country.string;countryOffset += country.byteCountWithEnd;}loc.area = readArea((int)countryOffset);} else if (redirectMode == REDIRECT_MODE_2) {loc.country = readString((int)read3ByteAsLong((int)offset + 5)).string;loc.area = readArea((int)offset + 8);} else {final QQwryString country = readString((int)offset + 4);loc.country = country.string;loc.area = readArea((int)offset + 4 + country.byteCountWithEnd);}return loc;} catch (Exception e) {return null;}}private String readArea(final int offset) {byte redirectMode = data[offset];if (redirectMode == REDIRECT_MODE_1 || redirectMode == REDIRECT_MODE_2) {long areaOffset = read3ByteAsLong((int)offset + 1);if (areaOffset == 0) {return "";} else {return readString((int)areaOffset).string;}} else {return readString(offset).string;}}private QQwryString readString(int offset) {int pos = offset;final byte[] b = new byte[128];int i;for (i = 0, b[i] = data[pos++]; b[i] != 0; b[++i] = data[pos++]);try{return new QQwryString(new String(b,0,i,"GBK"),i + 1);} catch(UnsupportedEncodingException e) {return new QQwryString("",0);}}/*** @Description:“.”号分隔的字符串转换为long类型的数字* @param ipStr * @return:long*/private static long inet_pton(String ipStr) {if(ipStr == null){throw new NullPointerException("ip不能为空");}String [] arr = ipStr.split("\\.");long ip = (Long.parseLong(arr[0])  & 0xFFL) << 24 & 0xFF000000L;ip |=  (Long.parseLong(arr[1])  & 0xFFL) << 16 & 0xFF0000L;ip |=  (Long.parseLong(arr[2])  & 0xFFL) << 8 & 0xFF00L;ip |=  (Long.parseLong(arr[3])  & 0xFFL);return ip;}private class QQwryString{public final String string;public final int byteCountWithEnd;public QQwryString(final String string,final int byteCountWithEnd) {this.string = string;this.byteCountWithEnd = byteCountWithEnd;}@Overridepublic String toString() {return string;}}
}

以上为主要代码，获取全部代码请点击全部代码

使用

final IPLocation ipLocation = new IPLocation(filePath);
Location loc = ipl.fetchIPLocation("182.92.240.50");
System.out.printf("%s %s",loc.country,loc.area);

格式改进

由于原格式中读取地区记录时采用重定向，有些繁琐。去掉之后格式更简单，国家和地区单独存放,索引里分别记录的国家和地区的地址。
新格式如下：

final IPFileConvertor convertor = new
IPFileConvertor(IPFileConvertor.class.getResource("/qqwry.dat").getPath(),"./qqwry.dat");
convertor.convert();
新格式使用方法和之前的一致，使用com.difeng.convert包下的解析类IPLocation解析即可。

相关连接：
qqwry下载： qqwry
全球ip地址库（收费）：IPLocation

原文来源：https://www.jianshu.com/p/01d3c19738c2