java maker 3.0 with IPv6 supported
This commit is contained in:
lion
parent
629f509c6f
commit
6727987960
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
<groupId>org.lionsoul</groupId>
|
||||
<artifactId>ip2region-maker</artifactId>
|
||||
<version>1.0.0</version>
|
||||
<version>3.0.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>ip2region</name>
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ package org.lionsoul.ip2region;
|
|||
import org.lionsoul.ip2region.xdb.IndexPolicy;
|
||||
import org.lionsoul.ip2region.xdb.Log;
|
||||
import org.lionsoul.ip2region.xdb.Maker;
|
||||
import org.lionsoul.ip2region.xdb.Version;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
|
|
@ -29,6 +30,7 @@ public class MakerTest {
|
|||
System.out.println("options:");
|
||||
System.out.println(" --src string source ip text file path");
|
||||
System.out.println(" --dst string destination binary xdb file path");
|
||||
System.out.println(" --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused");
|
||||
System.out.println(" --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7");
|
||||
System.out.println(" --log-level string set the log level, options: debug/info/warn/error");
|
||||
}
|
||||
|
|
@ -108,7 +110,7 @@ public class MakerTest {
|
|||
}
|
||||
|
||||
public static void genDb(String[] args) throws Exception {
|
||||
String srcFile = "", dstFile = "";
|
||||
String srcFile = "", dstFile = "", ipVersion = "";
|
||||
String fieldList = "", logLevel = "info";
|
||||
int indexPolicy = IndexPolicy.Vector;
|
||||
for (final String r : args) {
|
||||
|
|
@ -133,8 +135,10 @@ public class MakerTest {
|
|||
srcFile = val;
|
||||
} else if ("dst".equals(key)) {
|
||||
dstFile = val;
|
||||
} else if ("version".equals(key)) {
|
||||
ipVersion = val;
|
||||
} else if ("field-list".equals(key)) {
|
||||
fieldList = val;
|
||||
fieldList = val;
|
||||
} else if ("log-level".equals(key)) {
|
||||
logLevel = val;
|
||||
} else {
|
||||
|
|
@ -148,6 +152,9 @@ public class MakerTest {
|
|||
return;
|
||||
}
|
||||
|
||||
// IP version
|
||||
final Version version = Version.fromName(ipVersion);
|
||||
|
||||
final int[] fields = getFieldList(fieldList);
|
||||
if (fields == null) {
|
||||
return;
|
||||
|
|
@ -155,7 +162,7 @@ public class MakerTest {
|
|||
|
||||
// check and make the field list
|
||||
long tStart = System.currentTimeMillis();
|
||||
final Maker maker = new Maker(indexPolicy, srcFile, dstFile, fields);
|
||||
final Maker maker = new Maker(version, indexPolicy, srcFile, dstFile, fields);
|
||||
log.infof("Generating xdb with src=%s, dst=%s, logLevel=%s", srcFile, dstFile, logLevel);
|
||||
Maker.log.setLevel(logLevel);
|
||||
maker.init();
|
||||
|
|
|
|||
|
|
@ -1,79 +0,0 @@
|
|||
// Copyright 2022 The Ip2Region Authors. All rights reserved.
|
||||
// Use of this source code is governed by a Apache2.0-style
|
||||
// license that can be found in the LICENSE file.
|
||||
//
|
||||
// @Author Lion <chenxin619315@gmail.com>
|
||||
// @Date 2022/07/15
|
||||
|
||||
package org.lionsoul.ip2region;
|
||||
|
||||
import org.lionsoul.ip2region.xdb.IndexPolicy;
|
||||
import org.lionsoul.ip2region.xdb.Log;
|
||||
import org.lionsoul.ip2region.xdb.Segment;
|
||||
import org.lionsoul.ip2region.xdb.Util;
|
||||
|
||||
import java.util.regex.Matcher;
|
||||
|
||||
public class UtilTest {
|
||||
|
||||
public static final Log log = Log.getLogger(UtilTest.class);
|
||||
|
||||
public static void testSegment() {
|
||||
final String[] t_segs = {
|
||||
"1.1.0.0|1.3.3.24|中国|广东|深圳|电信",
|
||||
"0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP",
|
||||
"28.201.224.0|29.34.191.255|美国|0|0|0|0"
|
||||
};
|
||||
|
||||
for (String str : t_segs) {
|
||||
try {
|
||||
final Segment seg = Segment.parse(str);
|
||||
log.infof("segment(%s)=%s, split: ", str, seg.toString());
|
||||
for (Segment s : seg.split()) {
|
||||
log.debugf(s.toString());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.errorf("failed to parse segment `%s`: %s", str, e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void testFieldFilter() {
|
||||
final String reg = "0|16777215|保留|保留|保留|保留|||||||||||";
|
||||
final String[] fs = reg.split("\\|", -1);
|
||||
log.infof("length: %d", fs.length);
|
||||
for (String s : fs) {
|
||||
log.infof("%s", s);
|
||||
}
|
||||
}
|
||||
|
||||
public static void testIndexParse() {
|
||||
final String[] ss = new String[]{"1", "2-3", "23", "3-", "x-", "3-x", "4x"};
|
||||
for (String s : ss) {
|
||||
log.infof("parse: %s", s);
|
||||
final Matcher m = MakerTest.p.matcher(s);
|
||||
if (m.matches()) {
|
||||
for (int i = 0; i < m.groupCount(); i++) {
|
||||
log.infof("match: %d/%d, str: %s", i, m.groupCount(), m.group(i));
|
||||
}
|
||||
} else {
|
||||
log.infof("no match for: %s", s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println("+-- testing segments");
|
||||
testSegment();
|
||||
System.out.println();
|
||||
|
||||
System.out.println("+-- testing field filter");
|
||||
testFieldFilter();
|
||||
System.out.println();
|
||||
|
||||
System.out.println("+-- testing index parse");
|
||||
testIndexParse();
|
||||
System.out.println();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -19,6 +19,8 @@
|
|||
// -- 4bytes: generate unix timestamp (version)
|
||||
// -- 4bytes: index block start ptr
|
||||
// -- 4bytes: index block end ptr
|
||||
// -- 2bytes: ip version number (4/6 since IPv6 supporting)
|
||||
// -- 2bytes: runtime ptr bytes
|
||||
//
|
||||
//
|
||||
// 2. data block : region or whatever data info.
|
||||
|
|
@ -57,19 +59,24 @@ import java.util.*;
|
|||
|
||||
public class Maker {
|
||||
// constants define
|
||||
public static final int VersionNo = 2;
|
||||
public static final int HeaderInfoLength = 256;
|
||||
public static final int VectorIndexRows = 256;
|
||||
public static final int VectorIndexCols = 256;
|
||||
public static final int VectorIndexSize = 8;
|
||||
public static final int SegmentIndexSize = 14;
|
||||
public static final int VersionNo = 3; // 2 for XDB 2.0, 3 for XDB 3.0
|
||||
public static final int HeaderInfoLength = 256;
|
||||
public static final int VectorIndexRows = 256;
|
||||
public static final int VectorIndexCols = 256;
|
||||
public static final int VectorIndexSize = 8; // in bytes
|
||||
public static final int RuntimePtrSize = 4; // in bytes
|
||||
public static final long MaxFilePointer = (1L << (RuntimePtrSize * 8)) - 1;
|
||||
public static final int VectorIndexLength = VectorIndexRows * VectorIndexCols * VectorIndexSize;
|
||||
|
||||
public static final Log log = Log.getLogger(Maker.class);
|
||||
|
||||
// IP version since xdb 3.0
|
||||
private final Version version;
|
||||
|
||||
// source text file handle
|
||||
private final File srcFile;
|
||||
private final int[] fields;
|
||||
|
||||
private final List<Segment> segments;
|
||||
private final Charset bytesCharset;
|
||||
|
||||
|
|
@ -85,13 +92,14 @@ public class Maker {
|
|||
// vector index raw bytes
|
||||
private final byte[] vectorIndex;
|
||||
|
||||
public Maker(int policy, String srcPath, String dstPath, int[] fields) throws IOException {
|
||||
public Maker(Version version, int policy, String srcPath, String dstPath, int[] fields) throws IOException {
|
||||
this.srcFile = new File(srcPath);
|
||||
if (!this.srcFile.exists()) {
|
||||
throw new FileNotFoundException("source text file `" + srcPath + "` not found");
|
||||
}
|
||||
|
||||
this.fields = fields;
|
||||
this.version = version;
|
||||
this.fields = fields;
|
||||
|
||||
/// check and delete the target xdb file if it exists
|
||||
/// final File dstFile = new File(dstPath);
|
||||
|
|
@ -119,40 +127,31 @@ public class Maker {
|
|||
final byte[] header = new byte[HeaderInfoLength];
|
||||
|
||||
// encode the data
|
||||
Util.write(header, 0, VersionNo, 2);
|
||||
Util.write(header, 2, indexPolicy, 2);
|
||||
Util.write(header, 4, System.currentTimeMillis() / 1000, 4);
|
||||
Util.write(header, 8, 0, 4); // start index ptr
|
||||
Util.write(header, 12, 0, 4); // end index ptr
|
||||
// 1, data version number
|
||||
LittleEndian.put(header, 0, VersionNo, 2);
|
||||
|
||||
// 2, index policy code
|
||||
LittleEndian.put(header, 2, indexPolicy, 2);
|
||||
|
||||
// 3, generate unix timestamp
|
||||
LittleEndian.put(header, 4, System.currentTimeMillis() / 1000, 4);
|
||||
|
||||
// 4, index block start ptr
|
||||
LittleEndian.put(header, 8, 0, 4); // start index ptr
|
||||
|
||||
// 5, index block end ptr
|
||||
LittleEndian.put(header, 12, 0, 4); // end index ptr
|
||||
|
||||
// since xdb 3.0
|
||||
// 6, IP version
|
||||
LittleEndian.put(header, 16, version.id, 2); // IP version
|
||||
|
||||
// 7, runtime ptr bytes
|
||||
LittleEndian.put(header, 18, RuntimePtrSize, 2); // runtime ptr bytes
|
||||
|
||||
dstHandle.write(header);
|
||||
}
|
||||
|
||||
// internal method to apply the region fields filter
|
||||
private String getFilteredRegion(String region) {
|
||||
if (this.fields.length == 0) {
|
||||
return region;
|
||||
}
|
||||
|
||||
final String[] fs = region.split("\\|", -1);
|
||||
final StringBuilder sb = new StringBuilder();
|
||||
final int tailing = this.fields.length - 1;
|
||||
for (int i = 0; i < this.fields.length; i++) {
|
||||
final int idx = this.fields[i];
|
||||
if (idx >= fs.length) {
|
||||
throw new IllegalArgumentException("field index `"
|
||||
+ idx + "` exceeded the max length `" + fs.length + "`");
|
||||
}
|
||||
|
||||
sb.append(fs[idx]);
|
||||
if (sb.length() > 0 && i < tailing) {
|
||||
sb.append("|");
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
// load all the segments
|
||||
private void loadSegments() throws Exception {
|
||||
log.infof("try to load the segments ... ");
|
||||
|
|
@ -170,9 +169,9 @@ public class Maker {
|
|||
throw new Exception("invalid ip segment line `"+ps[0]+"`");
|
||||
}
|
||||
|
||||
final long sip = Util.checkIP(ps[0]);
|
||||
final long eip = Util.checkIP(ps[1]);
|
||||
if (sip > eip) {
|
||||
final byte[] sip = Util.parseIP(ps[0]);
|
||||
final byte[] eip = Util.parseIP(ps[1]);
|
||||
if (Util.ipCompare(sip, eip) > 0) {
|
||||
br.close();
|
||||
throw new Exception("start ip("+ps[0]+") should not be greater than end ip("+ps[1]+")");
|
||||
}
|
||||
|
|
@ -182,15 +181,21 @@ public class Maker {
|
|||
throw new Exception("empty region info in segment line `"+ps[2]+"`");
|
||||
}
|
||||
|
||||
// ip version check
|
||||
if (version.bytes != sip.length) {
|
||||
br.close();
|
||||
throw new InvalidInetAddressException("invalid ip segment(" + version.name + " expected)");
|
||||
}
|
||||
|
||||
// check the continuity of the data segment
|
||||
if (last != null) {
|
||||
if (last.endIP + 1 != sip) {
|
||||
if (Util.ipCompare(Util.ipAddOne(last.endIP), sip) != 0) {
|
||||
br.close();
|
||||
throw new Exception("discontinuous data segment: last.eip+1("+sip+") != seg.sip("+eip+", "+ps[0]+")");
|
||||
}
|
||||
}
|
||||
|
||||
final Segment seg = new Segment(sip, eip, getFilteredRegion(ps[2]));
|
||||
final Segment seg = new Segment(sip, eip, Util.regionFiltering(ps[2], this.fields));
|
||||
segments.add(seg);
|
||||
last = seg;
|
||||
}
|
||||
|
|
@ -209,16 +214,16 @@ public class Maker {
|
|||
}
|
||||
|
||||
// set the vector index info of the specified ip
|
||||
private void setVectorIndex(long ip, long ptr) {
|
||||
final int il0 = (int) ((ip >> 24) & 0xFF);
|
||||
final int il1 = (int) ((ip >> 16) & 0xFF);
|
||||
private void setVectorIndex(final byte[] ip, long ptr) {
|
||||
final int il0 = (int) (ip[0] & 0xFF);
|
||||
final int il1 = (int) (ip[1] & 0xFF);
|
||||
final int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
|
||||
final long sPtr = Util.getIntLong(vectorIndex, idx);
|
||||
final long sPtr = LittleEndian.getUint32(vectorIndex, idx);
|
||||
if (sPtr == 0) {
|
||||
Util.write(vectorIndex, idx, ptr, 4);
|
||||
Util.write(vectorIndex, idx + 4, ptr + SegmentIndexSize, 4);
|
||||
LittleEndian.put(vectorIndex, idx, ptr, 4);
|
||||
LittleEndian.put(vectorIndex, idx + 4, ptr + version.segmentIndexSize, 4);
|
||||
} else {
|
||||
Util.write(vectorIndex, idx + 4, ptr + SegmentIndexSize, 4);
|
||||
LittleEndian.put(vectorIndex, idx + 4, ptr + version.segmentIndexSize, 4);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -232,7 +237,7 @@ public class Maker {
|
|||
dstHandle.seek(HeaderInfoLength + VectorIndexLength);
|
||||
|
||||
log.infof("try to write the data block ... ");
|
||||
for (Segment seg : segments) {
|
||||
for (final Segment seg : segments) {
|
||||
log.debugf("try to write region `%s` ... ", seg.region);
|
||||
final DataEntry e = regionPool.get(seg.region);
|
||||
if (e != null) {
|
||||
|
|
@ -252,6 +257,11 @@ public class Maker {
|
|||
final long pos = dstHandle.getFilePointer();
|
||||
dstHandle.write(regionBuff);
|
||||
|
||||
// @TODO: remove this if the long ptr operation were supported
|
||||
if (pos >= MaxFilePointer) {
|
||||
throw new IOException("region ptr exceed the max length of '" + MaxFilePointer + "'");
|
||||
}
|
||||
|
||||
// record the mapping
|
||||
regionPool.put(seg.region, new DataEntry(regionBuff.length, pos));
|
||||
log.debugf(" --[Added] with ptr=%d", pos);
|
||||
|
|
@ -261,24 +271,34 @@ public class Maker {
|
|||
log.infof("try to write the segment index block ... ");
|
||||
int counter = 0;
|
||||
long startIndexPtr = -1, endIndexPtr = -1;
|
||||
final byte[] indexBuff = new byte[SegmentIndexSize]; // 4 + 4 + 2 + 4
|
||||
final byte[] indexBuff = new byte[version.segmentIndexSize];
|
||||
for (Segment seg : segments) {
|
||||
// we need the region ptr
|
||||
DataEntry e = regionPool.get(seg.region);
|
||||
final DataEntry e = regionPool.get(seg.region);
|
||||
if (e == null) {
|
||||
throw new Exception("missing ptr cache for region `"+seg.region+"`");
|
||||
}
|
||||
|
||||
int _offset = 0;
|
||||
List<Segment> segList = seg.split();
|
||||
log.debugf("try to index segment(%d splits) %s ... ", segList.size(), seg);
|
||||
for (Segment s : segList) {
|
||||
for (final Segment s : segList) {
|
||||
long pos = dstHandle.getFilePointer();
|
||||
|
||||
// encode the segment index info
|
||||
Util.write(indexBuff, 0, s.startIP, 4);
|
||||
Util.write(indexBuff, 4, s.endIP, 4);
|
||||
Util.write(indexBuff, 8, e.length, 2);
|
||||
Util.write(indexBuff, 10, e.ptr, 4);
|
||||
// @TODO: remove this if the long ptr operation were supported
|
||||
if (pos >= MaxFilePointer) {
|
||||
throw new IOException("region ptr exceed the max length of '" + MaxFilePointer + "'");
|
||||
}
|
||||
|
||||
// encode the segment index info.
|
||||
// @Note: in order to compatible with the old searcher implementation we choose to keep
|
||||
// encode the IPv4 bytes with little endian.
|
||||
// for IPv6 we choose the Network byte order (Big endian).
|
||||
version.putBytes(indexBuff, 0, s.startIP);
|
||||
version.putBytes(indexBuff, s.startIP.length, s.endIP);
|
||||
_offset = s.startIP.length + s.endIP.length;
|
||||
LittleEndian.put(indexBuff, _offset, e.length, 2);
|
||||
LittleEndian.put(indexBuff, _offset + 2, e.ptr, 4);
|
||||
dstHandle.write(indexBuff);
|
||||
|
||||
log.debugf("|-segment index: %d, ptr: %d, segment: %s", counter, pos, s);
|
||||
|
|
@ -301,8 +321,8 @@ public class Maker {
|
|||
|
||||
// 4, synchronize the segment index info
|
||||
log.infof("try to write the segment index ptr ... ");
|
||||
Util.write(indexBuff, 0, startIndexPtr, 4);
|
||||
Util.write(indexBuff, 4, endIndexPtr, 4);
|
||||
LittleEndian.put(indexBuff, 0, startIndexPtr, 4);
|
||||
LittleEndian.put(indexBuff, 4, endIndexPtr, 4);
|
||||
dstHandle.seek(8);
|
||||
dstHandle.write(indexBuff, 0, 8);
|
||||
|
||||
|
|
|
|||
|
|
@ -11,27 +11,11 @@ import java.util.ArrayList;
|
|||
import java.util.List;
|
||||
|
||||
public class Segment {
|
||||
public final long startIP;
|
||||
public final long endIP;
|
||||
public final byte[] startIP;
|
||||
public final byte[] endIP;
|
||||
public final String region;
|
||||
|
||||
// parser the Segment from an input string
|
||||
public static Segment parse(String input) throws Exception {
|
||||
final String[] ps = input.trim().split("\\|", 3);
|
||||
if (ps.length != 3) {
|
||||
throw new Exception("invalid ip segment `"+input+"`");
|
||||
}
|
||||
|
||||
long sip = Util.checkIP(ps[0]);
|
||||
long eip = Util.checkIP(ps[1]);
|
||||
if (sip > eip) {
|
||||
throw new Exception("start ip `"+ps[0]+"` should not be greater than end ip `"+ps[1]+"`");
|
||||
}
|
||||
|
||||
return new Segment(sip, eip, ps[2]);
|
||||
}
|
||||
|
||||
public Segment(long startIP, long endIP, String region) {
|
||||
public Segment(final byte[] startIP, final byte[] endIP, String region) {
|
||||
this.startIP = startIP;
|
||||
this.endIP = endIP;
|
||||
this.region = region;
|
||||
|
|
@ -39,39 +23,61 @@ public class Segment {
|
|||
|
||||
// split the current segment for vector index
|
||||
public List<Segment> split() {
|
||||
final long sByte1 = (int) ((startIP >> 24) & 0xFF);
|
||||
final long eByte1 = (int) ((endIP >> 24) & 0xFF);
|
||||
long nSip = startIP;
|
||||
final int sByte1 = (int) (startIP[0] & 0xFF);
|
||||
final int eByte1 = (int) (endIP[0] & 0xFF);
|
||||
final List<Segment> tList = new ArrayList<Segment>();
|
||||
for (long i = sByte1; i <= eByte1; i++) {
|
||||
long sip = (i << 24) | (nSip & 0xFFFFFF);
|
||||
long eip = (i << 24) | 0xFFFFFF;
|
||||
if (eip < endIP) {
|
||||
nSip = (i + 1) << 24;
|
||||
for (int i = sByte1; i <= eByte1; i++) {
|
||||
final byte[] sip = new byte[startIP.length];
|
||||
final byte[] eip = new byte[startIP.length];
|
||||
|
||||
if (i == sByte1) {
|
||||
System.arraycopy(startIP, 0, sip, 0, sip.length);
|
||||
} else {
|
||||
eip = endIP;
|
||||
sip[0] = (byte) (i & 0xFF);
|
||||
}
|
||||
|
||||
// append the new segment
|
||||
if (i == eByte1) {
|
||||
System.arraycopy(endIP, 0, eip, 0, eip.length);
|
||||
} else {
|
||||
eip[0] = (byte)(i & 0xFF);
|
||||
// fill the rest buffer with 0xFF
|
||||
for (int j = 1; j < eip.length; j++) {
|
||||
eip[j] = (byte) (0xFF);
|
||||
}
|
||||
}
|
||||
|
||||
// append the new segment:
|
||||
// @Note: Don't bother to copy the region.
|
||||
tList.add(new Segment(sip, eip, null));
|
||||
}
|
||||
|
||||
// 2, split the segments with the second byte
|
||||
final List<Segment> segList = new ArrayList<Segment>();
|
||||
for (Segment seg : tList) {
|
||||
final long base = seg.startIP & 0xFF000000;
|
||||
final long sb2 = (seg.startIP >> 16) & 0xFF;
|
||||
final long eb2 = (seg.endIP >> 16) & 0xFF;
|
||||
long tSip = seg.startIP;
|
||||
for (long i = sb2; i <= eb2; i++) {
|
||||
long sip = base | (i << 16) | (tSip & 0xFFFF);
|
||||
long eip = base | (i << 16) | 0xFFFF;
|
||||
if (eip < seg.endIP) {
|
||||
tSip = 0;
|
||||
final int sByte2 = (int) (seg.startIP[1] & 0xFF);
|
||||
final int eByte2 = (int) (seg.endIP[1] & 0xFF);
|
||||
for (int i = sByte2; i <= eByte2; i++) {
|
||||
final byte[] sip = new byte[seg.startIP.length];
|
||||
final byte[] eip = new byte[seg.startIP.length];
|
||||
sip[0] = seg.startIP[0];
|
||||
eip[0] = seg.startIP[0];
|
||||
|
||||
if (i == sByte2) {
|
||||
System.arraycopy(seg.startIP, 0, sip, 0, seg.startIP.length);
|
||||
} else {
|
||||
eip = seg.endIP;
|
||||
sip[1] = (byte) (i & 0xFF);
|
||||
}
|
||||
|
||||
if (i == eByte2) {
|
||||
System.arraycopy(seg.endIP, 0, eip, 0, seg.endIP.length);
|
||||
} else {
|
||||
eip[1] = (byte) (i & 0xFF);
|
||||
for (int j = 2; j < seg.endIP.length; j++) {
|
||||
eip[j] = (byte) 0xFF;
|
||||
}
|
||||
}
|
||||
|
||||
// append the new segment
|
||||
segList.add(new Segment(sip, eip, region));
|
||||
}
|
||||
}
|
||||
|
|
@ -80,6 +86,23 @@ public class Segment {
|
|||
}
|
||||
|
||||
@Override public String toString() {
|
||||
return Util.long2ip(startIP) + "|" + Util.long2ip(endIP) + "|" + region;
|
||||
return Util.ipToString(startIP) + "|" + Util.ipToString(endIP) + "|" + region;
|
||||
}
|
||||
|
||||
// parser the Segment from an input string
|
||||
public static Segment parse(String input) throws Exception {
|
||||
final String[] ps = input.trim().split("\\|", 3);
|
||||
if (ps.length != 3) {
|
||||
throw new Exception("invalid ip segment `"+input+"`");
|
||||
}
|
||||
|
||||
final byte[] sip = Util.parseIP(ps[0]);
|
||||
final byte[] eip = Util.parseIP(ps[1]);
|
||||
if (Util.ipCompare(sip, eip) > 0) {
|
||||
throw new Exception("start ip `"+ps[0]+"` should not be greater than end ip `"+ps[1]+"`");
|
||||
}
|
||||
|
||||
return new Segment(sip, eip, ps[2]);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,15 +24,15 @@ public class Util
|
|||
}
|
||||
|
||||
// convert the byte[] ip to string ip address
|
||||
public static String ipToString(final byte[] ip) throws InvalidInetAddressException {
|
||||
public static String ipToString(final byte[] ip) {
|
||||
if (ip.length != 4 && ip.length != 16) {
|
||||
throw new InvalidInetAddressException("invalid ip address length `"+ip.length+"`");
|
||||
return String.format("invalid-ip-address-length: %d", ip.length);
|
||||
}
|
||||
|
||||
try {
|
||||
return InetAddress.getByAddress(ip).getHostAddress();
|
||||
} catch (UnknownHostException e) {
|
||||
throw new InvalidInetAddressException("invalid ip address `"+ipArrayString(ip)+"`");
|
||||
return String.format("invalid-ip-address `%s`", ipArrayString(ip));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -101,65 +101,29 @@ public class Util
|
|||
return r;
|
||||
}
|
||||
|
||||
|
||||
// write specified bytes into a byte array start from offset
|
||||
public static void write( byte[] b, int offset, long v, int bytes) {
|
||||
for ( int i = 0; i < bytes; i++ ) {
|
||||
b[offset++] = (byte)((v >>> (8 * i)) & 0xFF);
|
||||
}
|
||||
}
|
||||
|
||||
// write a int to a byte array
|
||||
public static void writeIntLong(byte[] b, int offset, long v) {
|
||||
b[offset++] = (byte)((v ) & 0xFF);
|
||||
b[offset++] = (byte)((v >> 8) & 0xFF);
|
||||
b[offset++] = (byte)((v >> 16) & 0xFF);
|
||||
b[offset ] = (byte)((v >> 24) & 0xFF);
|
||||
}
|
||||
|
||||
// get an int from a byte array start from the specified offset
|
||||
public static long getIntLong(byte[] b, int offset) {
|
||||
return (
|
||||
((b[offset++] & 0x000000FFL)) |
|
||||
((b[offset++] << 8) & 0x0000FF00L) |
|
||||
((b[offset++] << 16) & 0x00FF0000L) |
|
||||
((b[offset ] << 24) & 0xFF000000L)
|
||||
);
|
||||
}
|
||||
|
||||
public static int getInt2(byte[] b, int offset) {
|
||||
return (
|
||||
((b[offset++]) & 0x000000FF) |
|
||||
((b[offset ] << 8) & 0x0000FF00)
|
||||
);
|
||||
}
|
||||
|
||||
/* long int to ip string */
|
||||
public static String long2ip( long ip ) {
|
||||
return String.valueOf((ip >> 24) & 0xFF) + '.' +
|
||||
((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF);
|
||||
}
|
||||
|
||||
public static final byte[] shiftIndex = {24, 16, 8, 0};
|
||||
|
||||
/* check the specified ip address */
|
||||
public static long checkIP(String ip) throws Exception {
|
||||
final String[] ps = ip.split("\\.");
|
||||
if (ps.length != 4) {
|
||||
throw new Exception("invalid ip address `" + ip + "`");
|
||||
// region filtering
|
||||
public static String regionFiltering(String region, int[] fields) {
|
||||
if (fields.length == 0) {
|
||||
return region;
|
||||
}
|
||||
|
||||
long ipDst = 0;
|
||||
for (int i = 0; i < ps.length; i++) {
|
||||
int val = Integer.parseInt(ps[i]);
|
||||
if (val > 255) {
|
||||
throw new Exception("ip part `"+ps[i]+"` should be less then 256");
|
||||
final String[] fs = region.split("\\|", -1);
|
||||
final StringBuilder sb = new StringBuilder();
|
||||
final int tailing = fields.length - 1;
|
||||
for (int i = 0; i < fields.length; i++) {
|
||||
final int idx = fields[i];
|
||||
if (idx >= fs.length) {
|
||||
throw new IllegalArgumentException("field index `"
|
||||
+ idx + "` exceeded the max length `" + fs.length + "`");
|
||||
}
|
||||
|
||||
ipDst |= ((long) val << shiftIndex[i]);
|
||||
sb.append(fs[idx]);
|
||||
if (sb.length() > 0 && i < tailing) {
|
||||
sb.append("|");
|
||||
}
|
||||
}
|
||||
|
||||
return ipDst & 0xFFFFFFFFL;
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import org.junit.Test;
|
|||
|
||||
public class LittleEndianTest {
|
||||
|
||||
private static final Log log = Log.getLogger(LittleEndianTest.class).setLevel(Log.DEBUG);
|
||||
|
||||
@Test
|
||||
public void testAll() {
|
||||
final byte[] buff = new byte[14];
|
||||
|
|
@ -25,10 +27,10 @@ public class LittleEndianTest {
|
|||
assertEquals(LittleEndian.getInt2(buff, 8), 24);
|
||||
assertEquals(LittleEndian.getUint32(buff, 10), 1024);
|
||||
|
||||
System.out.printf("uint32(buff, 0): %d\n", LittleEndian.getUint32(buff, 0));
|
||||
System.out.printf("uint32(buff, 4): %d\n", LittleEndian.getUint32(buff, 4));
|
||||
System.out.printf("int2(buff, 8): %d\n", LittleEndian.getInt2(buff, 8));
|
||||
System.out.printf("uint32(buff, 10): %d\n", LittleEndian.getUint32(buff, 10));
|
||||
log.debugf("uint32(buff, 0): %d", LittleEndian.getUint32(buff, 0));
|
||||
log.debugf("uint32(buff, 4): %d", LittleEndian.getUint32(buff, 4));
|
||||
log.debugf("int2(buff, 8): %d", LittleEndian.getInt2(buff, 8));
|
||||
log.debugf("uint32(buff, 10): %d", LittleEndian.getUint32(buff, 10));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
package org.lionsoul.ip2region.xdb;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
public class SegmentTest {
|
||||
|
||||
private final static Log log = Log.getLogger(SegmentTest.class).setLevel(Log.DEBUG);
|
||||
|
||||
@Test
|
||||
public void testParse() throws Exception {
|
||||
final String[] strs = {
|
||||
"1.1.0.0|1.3.3.24|中国|广东|深圳|电信",
|
||||
"28.201.224.0|29.34.191.255|美国|0|0|0|0",
|
||||
"2001:4:112::|2001:4:112:ffff:ffff:ffff:ffff:ffff|德国|黑森|美因河畔法兰克福|专线用户"
|
||||
};
|
||||
|
||||
for (final String str : strs) {
|
||||
final Segment seg = Segment.parse(str);
|
||||
log.debugf("seg: %s", seg.toString());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSplit() throws Exception {
|
||||
final String[] t_segs = {
|
||||
"1.1.0.0|1.3.3.24|中国|广东|深圳|电信",
|
||||
"28.201.224.0|29.34.191.255|美国|0|0|0|0",
|
||||
"fec0::|ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||专线用户|IANA"
|
||||
};
|
||||
|
||||
for (String str : t_segs) {
|
||||
final Segment seg = Segment.parse(str);
|
||||
log.infof("segment(%s)->split: ", seg.toString());
|
||||
for (final Segment s : seg.split()) {
|
||||
log.debugf(s.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -4,7 +4,7 @@ import org.junit.Test;
|
|||
|
||||
public class UtilTest {
|
||||
|
||||
public static final Log log = Log.getLogger(UtilTest.class).setLevel(Log.DEBUG);
|
||||
private static final Log log = Log.getLogger(UtilTest.class).setLevel(Log.DEBUG);
|
||||
|
||||
@Test
|
||||
public void testCheckIP() throws InvalidInetAddressException {
|
||||
|
|
@ -75,4 +75,18 @@ public class UtilTest {
|
|||
log.debugf("ipSubOne(%s): %s", ip, Util.ipToString(Util.ipSubOne(ipBytes)));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRegionFiltering() {
|
||||
final String[] regions = new String[]{
|
||||
"亚洲|中国|广东|深圳|宝安|电信|113.88311|22.55371|440306|0755|518100|Asia/Shanghai|CNY|11|CHXX0120",
|
||||
"大洲|国家|省份|城市|区县|ISP|经度|纬度|0|0|0|0|0"
|
||||
};
|
||||
|
||||
final int[] fields = new int[] {1,2,3,4,6,7};
|
||||
for (String region : regions) {
|
||||
log.infof("filtering: %s", Util.regionFiltering(region, fields));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@ import static org.junit.Assert.assertEquals;
|
|||
import org.junit.Test;
|
||||
|
||||
public class VersionTest {
|
||||
|
||||
private static final Log log = Log.getLogger(VersionTest.class).setLevel(Log.DEBUG);
|
||||
|
||||
@Test
|
||||
public void testFromName() throws Exception {
|
||||
final String[] vers = new String[]{"IPv4", "IPv6"};
|
||||
|
|
@ -13,7 +16,7 @@ public class VersionTest {
|
|||
final Version v6 = Version.fromName(vers[1]);
|
||||
assertEquals(v6.name, vers[1]);
|
||||
|
||||
System.out.printf("v4: %s\n", v4);
|
||||
System.out.printf("v6: %s\n", v6);
|
||||
log.debugf("v4: %s", v4);
|
||||
log.debugf("v6: %s", v6);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue