From 285a06b9cfa7fabd15197731df49bd3730042860 Mon Sep 17 00:00:00 2001 From: Lion Date: Tue, 12 Jul 2022 12:07:02 +0800 Subject: [PATCH 1/7] java xdb maker --- .../org/lionsoul/ip2region/MakerTest.java | 11 + .../org/lionsoul/ip2region/xdb/Maker.java | 11 + maker/java/pom.xml | 190 ++++++++++++++++++ 3 files changed, 212 insertions(+) create mode 100644 maker/java/org/lionsoul/ip2region/MakerTest.java create mode 100644 maker/java/org/lionsoul/ip2region/xdb/Maker.java create mode 100644 maker/java/pom.xml diff --git a/maker/java/org/lionsoul/ip2region/MakerTest.java b/maker/java/org/lionsoul/ip2region/MakerTest.java new file mode 100644 index 0000000..4639393 --- /dev/null +++ b/maker/java/org/lionsoul/ip2region/MakerTest.java @@ -0,0 +1,11 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// +// @Author Lion +// @Date 2022/07/12 + +package org.lionsoul.ip2region; + +public class MakerTest { +} diff --git a/maker/java/org/lionsoul/ip2region/xdb/Maker.java b/maker/java/org/lionsoul/ip2region/xdb/Maker.java new file mode 100644 index 0000000..b28c829 --- /dev/null +++ b/maker/java/org/lionsoul/ip2region/xdb/Maker.java @@ -0,0 +1,11 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// +// @Author Lion +// @Date 2022/07/12 + +package org.lionsoul.ip2region.xdb; + +public class Maker { +} diff --git a/maker/java/pom.xml b/maker/java/pom.xml new file mode 100644 index 0000000..391aef7 --- /dev/null +++ b/maker/java/pom.xml @@ -0,0 +1,190 @@ + + + 4.0.0 + + org.lionsoul + ip2region + 2.6.4 + jar + + ip2region + https://github.com/lionsoul2014/ip2region + Open source offline internet address db manager framework and locator + + + + The Apache Software License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + git@github.com:lionsoul2014/ip2region.git + scm:git:git@github.com:lionsoul2014/ip2region.git + scm:git:git@github.com:lionsoul2014/ip2region.git + + + + + lionsoul + chenxin + chenxin619315@gmail.com + + + + + https://github.com/lionsoul2014/ip2region/issues + Github issues + + + + UTF-8 + UTF-8 + + + + + junit + junit + 4.13.1 + test + + + + + + + org.apache.maven.plugins + maven-source-plugin + 2.1.2 + + + attach-sources + package + + jar + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.9 + + + attach-javadocs + package + + jar + + + ${javadoc.opts} + + + + + false + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + + + org.lionsoul.ip2region.MakerTest + + + + + + + + + + + + java8-doclint-disabled + + [1.8,) + + + -Xdoclint:none + + + + release + + + + + org.apache.maven.plugins + maven-source-plugin + 2.2.1 + + + package + + jar-no-fork + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.9.1 + + + package + + jar + + + ${javadoc.opts} + + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.5 + + + verify + + sign + + + + + + + + + oss + https://oss.sonatype.org/content/repositories/snapshots/ + + + oss + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + + + From 3bec5da60af3da409f26fc8793b01d06b400a407 Mon Sep 17 00:00:00 2001 From: lion Date: Wed, 13 Jul 2022 16:47:31 +0800 Subject: [PATCH 2/7] stdlize the maven project --- .../main/java}/org/lionsoul/ip2region/MakerTest.java | 5 +++++ .../main/java}/org/lionsoul/ip2region/xdb/Maker.java | 0 2 files changed, 5 insertions(+) rename maker/java/{ => src/main/java}/org/lionsoul/ip2region/MakerTest.java (76%) rename maker/java/{ => src/main/java}/org/lionsoul/ip2region/xdb/Maker.java (100%) diff --git a/maker/java/org/lionsoul/ip2region/MakerTest.java b/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java similarity index 76% rename from maker/java/org/lionsoul/ip2region/MakerTest.java rename to maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java index 4639393..017facd 100644 --- a/maker/java/org/lionsoul/ip2region/MakerTest.java +++ b/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java @@ -8,4 +8,9 @@ package org.lionsoul.ip2region; public class MakerTest { + + public static void main(String[] args) { + System.out.println("xdb maker"); + } + } diff --git a/maker/java/org/lionsoul/ip2region/xdb/Maker.java b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java similarity index 100% rename from maker/java/org/lionsoul/ip2region/xdb/Maker.java rename to maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java From 4ab63d3caaa928b3c293a2733f7d35d36437f105 Mon Sep 17 00:00:00 2001 From: lion Date: Wed, 13 Jul 2022 17:00:24 +0800 Subject: [PATCH 3/7] maker constants define --- .../org/lionsoul/ip2region/xdb/Maker.java | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java index b28c829..23f0d47 100644 --- a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java +++ b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java @@ -5,7 +5,62 @@ // @Author Lion // @Date 2022/07/12 +// --- Ip2Region v2.0 data structure +// +// +----------------+--------------------------+---------------+--------------+ +// | header space | vector speed up index | data payload | block index | +// +----------------+--------------------------+---------------+--------------+ +// | 256 bytes | 512 KiB (fixed) | dynamic size | dynamic size | +// +----------------+--------------------------+---------------+--------------+ +// +// 1. padding space : for header info like block index ptr, version, release date eg ... or any other temporary needs. +// -- 2bytes: version number, different version means structure update, it fixed to 2 for now +// -- 2bytes: index algorithm code. +// -- 4bytes: generate unix timestamp (version) +// -- 4bytes: index block start ptr +// -- 4bytes: index block end ptr +// +// +// 2. data block : region or whatever data info. +// 3. segment index block : binary index block. +// 4. vector index block : fixed index info for block index search speed up. +// space structure table: +// -- 0 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block +// -- 1 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block +// -- 2 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block +// -- ... +// -- 255 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block +// +// +// super block structure: +// +-----------------------+----------------------+ +// | first index block ptr | last index block ptr | +// +-----------------------+----------------------+ +// +// data entry structure: +// +--------------------+-----------------------+ +// | 2bytes (for desc) | dynamic length | +// +--------------------+-----------------------+ +// data length whatever in bytes +// +// index entry structure +// +------------+-----------+---------------+------------+ +// | 4bytes | 4bytes | 2bytes | 4 bytes | +// +------------+-----------+---------------+------------+ +// start ip end ip data length data ptr + package org.lionsoul.ip2region.xdb; public class Maker { + // constants define + public static final int VersionNo = 2; + public static final int HeaderInfoLength = 256; + public static final int VectorIndexRows = 256; + public static final int VectorIndexCols = 256; + public static final int VectorIndexSize = 8; + public static final int SegmentIndexSize = 14; + + public Maker() { + } + } From 71404d8c342be1dceade048e9687158cfb5add55 Mon Sep 17 00:00:00 2001 From: lion Date: Thu, 14 Jul 2022 16:38:26 +0800 Subject: [PATCH 4/7] maker segment and index policy manager --- .../lionsoul/ip2region/xdb/IndexPolicy.java | 25 ++++ .../java/org/lionsoul/ip2region/xdb/Log.java | 116 ++++++++++++++++++ .../org/lionsoul/ip2region/xdb/Maker.java | 76 +++++++++++- .../org/lionsoul/ip2region/xdb/Segment.java | 85 +++++++++++++ .../java/org/lionsoul/ip2region/xdb/Util.java | 72 +++++++++++ 5 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 maker/java/src/main/java/org/lionsoul/ip2region/xdb/IndexPolicy.java create mode 100644 maker/java/src/main/java/org/lionsoul/ip2region/xdb/Log.java create mode 100644 maker/java/src/main/java/org/lionsoul/ip2region/xdb/Segment.java create mode 100644 maker/java/src/main/java/org/lionsoul/ip2region/xdb/Util.java diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/IndexPolicy.java b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/IndexPolicy.java new file mode 100644 index 0000000..64ddc11 --- /dev/null +++ b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/IndexPolicy.java @@ -0,0 +1,25 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// +// @Author Lion +// @Date 2022/07/14 + +package org.lionsoul.ip2region.xdb; + +public class IndexPolicy { + public static final int Vector = 1; + public static final int BTree = 2; + + // parser the index policy from string + public static int parse(String policy) throws Exception { + String v = policy.toLowerCase(); + if ("vector".equals(v)) { + return Vector; + } else if ("btree".equals(v)) { + return BTree; + } else { + throw new Exception("unknown index policy `"+policy+"`"); + } + } +} \ No newline at end of file diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Log.java b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Log.java new file mode 100644 index 0000000..d3a3856 --- /dev/null +++ b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Log.java @@ -0,0 +1,116 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// +// @Author Lion +// @Date 2022/07/14 + +package org.lionsoul.ip2region.xdb; + +import java.text.SimpleDateFormat; +import java.util.Date; + +// simple log implementation +public class Log { + + /* Log level constants define */ + public static final int DEBUG = 0; + public static final int INFO = 1; + public static final int WARN = 2; + public static final int ERROR = 3; + + // level name + public static final String[] level_string = new String[] { + "DEBUG", + "INFO", + "WARN", + "ERROR" + }; + + public final Class baseClass; + private static int level; + + public Log(Class baseClass) { + this.baseClass = baseClass; + } + + public static Log getLogger(Class baseClass) { + return new Log(baseClass); + } + + public String format(int level, String format, Object... args) { + // append the datetime + final StringBuilder sb = new StringBuilder(); + final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + sb.append(String.format("%s %-5s ", sdf.format(new Date()), level_string[level])); + + // append the class name + sb.append(baseClass.getName()).append(' '); + sb.append(String.format(format, args)); + return sb.toString(); + } + + public void printf(int level, String format, Object... args) { + if (level < DEBUG || level > ERROR) { + throw new IndexOutOfBoundsException("invalid level index " + level); + } + + // level filter + if (level < Log.level) { + return; + } + + System.out.println(format(level, format, args)); + System.out.flush(); + } + + public String getDebugf(String format, Object... args) { + return format(DEBUG, format, args); + } + + public void debugf(String format, Object... args) { + printf(DEBUG, format, args); + } + + public String getInfof(String format, Object... args) { + return format(INFO, format, args); + } + + public void infof(String format, Object... args) { + printf(INFO, format, args); + } + + public String getWarnf(String format, Object... args) { + return format(WARN, format, args); + } + + public void warnf(String format, Object... args) { + printf(WARN, format, args); + } + + public String getErrorf(String format, Object... args) { + return format(ERROR, format, args); + } + + public void errorf(String format, Object... args) { + printf(ERROR, format, args); + } + + public static void setLevel(int level) { + Log.level = level; + } + + public static void setLevel(String level) { + String v = level.toLowerCase(); + if ("debug".equals(v)) { + Log.level = DEBUG; + } else if ("info".equals(v)) { + Log.level = INFO; + } else if ("warn".equals(v)) { + Log.level = WARN; + } else if ("error".equals(v)) { + Log.level = ERROR; + } + } + +} \ No newline at end of file diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java index 23f0d47..d86f427 100644 --- a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java +++ b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java @@ -51,6 +51,13 @@ package org.lionsoul.ip2region.xdb; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.HashMap; +import java.util.Map; + public class Maker { // constants define public static final int VersionNo = 2; @@ -60,7 +67,74 @@ public class Maker { public static final int VectorIndexSize = 8; public static final int SegmentIndexSize = 14; - public Maker() { + private static final Log log = Log.getLogger(Maker.class); + + // source text file handle + private final File srcFile; + + // destination binary file handle + private final RandomAccessFile dstHandle; + + // index policy + private final int indexPolicy; + + // region pool + private final Map regionPool; + + // vector index raw bytes + private byte[] vectorIndex; + + public Maker(int policy, String srcFile, String dstFile) throws FileNotFoundException { + this.srcFile = new File(srcFile); + if (!this.srcFile.exists()) { + throw new FileNotFoundException("source text file `" +srcFile+ "` not found"); + } + + this.dstHandle = new RandomAccessFile(dstFile, "r"); + this.indexPolicy = policy; + this.regionPool = new HashMap(); + } + + // init the header of the target xdb binary file + private void initHeader() throws IOException { + log.infof("try to init the db header ... "); + dstHandle.seek(0); + + // make and write the header space + final byte[] header = new byte[HeaderInfoLength]; + + // encode the data + Util.write(header, 0, VersionNo, 2); + Util.write(header, 2, indexPolicy, 2); + Util.write(header, 4, System.currentTimeMillis() / 1000, 4); + Util.write(header, 8, 0, 4); // start index ptr + Util.write(header, 12, 0, 4); // end index ptr + + dstHandle.write(header); + } + + // load all the segments + private void loadSegments() { + + } + + // init the maker + public void init() throws IOException { + // init the db header + initHeader(); + + // load all the segments + loadSegments(); + } + + // start to make the binary file + public void make() { + + } + + // end the make, do the resource clean up + public void end() throws IOException { + this.dstHandle.close(); } } diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Segment.java b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Segment.java new file mode 100644 index 0000000..43fdf50 --- /dev/null +++ b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Segment.java @@ -0,0 +1,85 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// +// @Author Lion +// @Date 2022/07/14 + +package org.lionsoul.ip2region.xdb; + +import java.util.ArrayList; +import java.util.List; + +public class Segment { + public final long startIP; + public final long endIP; + public final String region; + + // parser the Segment from an input string + public static Segment parse(String input) throws Exception { + String[] ps = input.split("\\|", 3); + if (ps.length != 3) { + throw new Exception("invalid ip segment `"+input+"`"); + } + + long sip = Util.checkIP(ps[0]); + long eip = Util.checkIP(ps[1]); + if (sip > eip) { + throw new Exception("start ip `"+ps[0]+"` should not be greater than end ip `"+ps[1]+"`"); + } + + return new Segment(sip, eip, ps[2]); + } + + public Segment(long startIP, long endIP, String region) { + this.startIP = startIP; + this.endIP = endIP; + this.region = region; + } + + // split the current segment for vector index + public List split() { + long sByte1 = (int) ((startIP >> 24) & 0xFF); + long eByte1 = (int) ((endIP >> 24) & 0xFF); + long nSip = startIP; + final List tList = new ArrayList(); + for (long i = sByte1; i <= eByte1; i++) { + long sip = (i << 24) | (nSip & 0xFFFFFF); + long eip = (i << 24) | 0xFFFFFF; + if (eip < endIP) { + nSip = (i + 1) << 24; + } else { + eip = endIP; + } + + // append the new segment + tList.add(new Segment(sip, eip, null)); + } + + // 2, split the segments with the second byte + final List segList = new ArrayList(); + for (Segment seg : tList) { + long base = seg.startIP & 0xFF000000; + long tSip = seg.startIP; + long sb2 = (seg.startIP >> 16) & 0xFF; + long eb2 = (seg.endIP >> 16) & 0xFF; + for (long i = sb2; i <= eb2; i++) { + long sip = base | (i << 16) | (tSip & 0xFFFF); + long eip = base | (i << 16) | 0xFFFF; + if (eip < seg.endIP) { + tSip = 0; + } else { + eip = seg.endIP; + } + + segList.add(new Segment(sip, eip, seg.region)); + } + } + + return segList; + } + + @Override public String toString() { + return Util.long2ip(startIP) + "|" + Util.long2ip(endIP) + "|" + region; + } +} diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Util.java b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Util.java new file mode 100644 index 0000000..33d6f5a --- /dev/null +++ b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Util.java @@ -0,0 +1,72 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// +// @Author Lion +// @Date 2022/07/14 + +package org.lionsoul.ip2region.xdb; + +public class Util +{ + // write specified bytes into a byte array start from offset + public static void write( byte[] b, int offset, long v, int bytes) { + for ( int i = 0; i < bytes; i++ ) { + b[offset++] = (byte)((v >>> (8 * i)) & 0xFF); + } + } + + // write a int to a byte array + public static void writeIntLong(byte[] b, int offset, long v) { + b[offset++] = (byte)((v ) & 0xFF); + b[offset++] = (byte)((v >> 8) & 0xFF); + b[offset++] = (byte)((v >> 16) & 0xFF); + b[offset ] = (byte)((v >> 24) & 0xFF); + } + + // get an int from a byte array start from the specified offset + public static long getIntLong(byte[] b, int offset) { + return ( + ((b[offset++] & 0x000000FFL)) | + ((b[offset++] << 8) & 0x0000FF00L) | + ((b[offset++] << 16) & 0x00FF0000L) | + ((b[offset ] << 24) & 0xFF000000L) + ); + } + + public static int getInt2(byte[] b, int offset) { + return ( + (b[offset++] & 0x000000FF) | + (b[offset ] & 0x0000FF00) + ); + } + + /* long int to ip string */ + public static String long2ip( long ip ) { + return String.valueOf((ip >> 24) & 0xFF) + '.' + + ((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF); + } + + public static final byte[] shiftIndex = {24, 16, 8, 0}; + + /* check the specified ip address */ + public static long checkIP(String ip) throws Exception { + String[] ps = ip.split("\\."); + if (ps.length != 4) { + throw new Exception("invalid ip address `" + ip + "`"); + } + + long ipDst = 0; + for (int i = 0; i < ps.length; i++) { + int val = Integer.parseInt(ps[i]); + if (val > 255) { + throw new Exception("ip part `"+ps[i]+"` should be less then 256"); + } + + ipDst |= ((long) val << shiftIndex[i]); + } + + return ipDst & 0xFFFFFFFFL; + } + +} From 01529b888700716bc956c6a202a8040f9672d946 Mon Sep 17 00:00:00 2001 From: Lion Date: Fri, 15 Jul 2022 11:50:07 +0800 Subject: [PATCH 5/7] fix the bug of segment splits and tested --- .../java/org/lionsoul/ip2region/UtilTest.java | 75 +++++++++++++++++++ .../org/lionsoul/ip2region/xdb/Segment.java | 2 +- 2 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 maker/java/src/main/java/org/lionsoul/ip2region/UtilTest.java diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/UtilTest.java b/maker/java/src/main/java/org/lionsoul/ip2region/UtilTest.java new file mode 100644 index 0000000..7569b1b --- /dev/null +++ b/maker/java/src/main/java/org/lionsoul/ip2region/UtilTest.java @@ -0,0 +1,75 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// +// @Author Lion +// @Date 2022/07/15 + +package org.lionsoul.ip2region; + +import org.lionsoul.ip2region.xdb.IndexPolicy; +import org.lionsoul.ip2region.xdb.Log; +import org.lionsoul.ip2region.xdb.Segment; +import org.lionsoul.ip2region.xdb.Util; + +public class UtilTest { + + public static final Log log = Log.getLogger(UtilTest.class); + + public static void testIndexPolicy() { + final String[] inputs = {"vector", "btree", "VecTor", "BTree", "abc"}; + for (String str : inputs) { + try { + int policy = IndexPolicy.parse(str); + log.infof("parse(%s)=%d", str, policy); + } catch (Exception e) { + log.errorf("parse index policy `%s`: %s", str, e.getMessage()); + } + } + } + + public static void testIPUtil() { + final String ipStr = "1.2.3.4"; + try { + long ip = Util.checkIP(ipStr); + log.infof("checkIP(%s)=%d, validate: %s, long2ip(%d)=%s", ipStr, ip, ip == 16909060 ? "true" : "false", ip, Util.long2ip(ip)); + } catch (Exception e) { + log.errorf("failed to check ip `%s`: %s", ipStr, e.getMessage()); + } + } + + public static void testSegment() { + final String[] t_segs = { + "1.1.0.0|1.3.3.24|中国|广东|深圳|电信", + "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP", + "28.201.224.0|29.34.191.255|美国|0|0|0|0" + }; + + for (String str : t_segs) { + try { + final Segment seg = Segment.parse(str); + log.infof("segment(%s)=%s, split: ", str, seg.toString()); + for (Segment s : seg.split()) { + log.debugf(s.toString()); + } + } catch (Exception e) { + log.errorf("failed to parse segment `%s`: %s", str, e.getMessage()); + } + } + } + + public static void main(String[] args) { + System.out.println("+-- testing index policy"); + testIndexPolicy(); + System.out.println(); + + System.out.println("+-- testing ip util"); + testIPUtil(); + System.out.println(); + + System.out.println("+-- testing segments"); + testSegment(); + System.out.println(); + } + +} diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Segment.java b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Segment.java index 43fdf50..61a350b 100644 --- a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Segment.java +++ b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Segment.java @@ -72,7 +72,7 @@ public class Segment { eip = seg.endIP; } - segList.add(new Segment(sip, eip, seg.region)); + segList.add(new Segment(sip, eip, region)); } } From 4d3d5501b532dc20c6407e2f03141ec100a5c528 Mon Sep 17 00:00:00 2001 From: Lion Date: Fri, 15 Jul 2022 19:05:42 +0800 Subject: [PATCH 6/7] maker and maker test implemented --- maker/java/pom.xml | 14 +- .../org/lionsoul/ip2region/MakerTest.java | 76 +++++++- .../org/lionsoul/ip2region/xdb/Maker.java | 179 ++++++++++++++++-- 3 files changed, 253 insertions(+), 16 deletions(-) diff --git a/maker/java/pom.xml b/maker/java/pom.xml index 391aef7..1bf6653 100644 --- a/maker/java/pom.xml +++ b/maker/java/pom.xml @@ -3,8 +3,8 @@ 4.0.0 org.lionsoul - ip2region - 2.6.4 + ip2region-maker + 1.0.0 jar ip2region @@ -47,7 +47,7 @@ junit junit - 4.13.1 + 4.13.2 test @@ -110,6 +110,14 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + 6 + 6 + + diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java b/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java index 017facd..414a2da 100644 --- a/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java +++ b/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java @@ -7,10 +7,84 @@ package org.lionsoul.ip2region; +import org.lionsoul.ip2region.xdb.IndexPolicy; +import org.lionsoul.ip2region.xdb.Log; +import org.lionsoul.ip2region.xdb.Maker; + public class MakerTest { + public final static Log log = Log.getLogger(MakerTest.class); + + public static void printHelp(String[] args) { + System.out.println("ip2region xdb maker"); + System.out.println("java -jar ip2region-maker-{version}.jar [command options]"); + System.out.println("options:"); + System.out.println(" --src string source ip text file path"); + System.out.println(" --dst string destination binary xdb file path"); + } + + public static void genDb(String[] args) throws Exception { + String srcFile = "", dstFile = ""; + int indexPolicy = IndexPolicy.Vector; + for (final String r : args) { + if (r.length() < 5) { + continue; + } + + if (r.indexOf("--") != 0) { + continue; + } + + int sIdx = r.indexOf('='); + if (sIdx < 0) { + System.out.printf("missing = for args pair `%s`\n", r); + return; + } + + String key = r.substring(2, sIdx); + String val = r.substring(sIdx + 1); + // System.out.printf("key=%s, val=%s\n", key, val); + if ("src".equals(key)) { + srcFile = val; + } else if ("dst".equals(key)) { + dstFile = val; + } else if ("index".equals(key)) { + try { + indexPolicy = IndexPolicy.parse(val); + } catch (Exception e) { + System.out.println("parse policy " + e); + } + } else { + System.out.printf("undefined option `%s`\n", r); + return; + } + } + + if (srcFile.length() < 1 || dstFile.length() < 1) { + printHelp(args); + return; + } + + long tStart = System.currentTimeMillis(); + Maker maker = new Maker(indexPolicy, srcFile, dstFile); + maker.init(); + maker.start(); + maker.end(); + + log.infof("Done, elapsed: %d s\n", (System.currentTimeMillis() - tStart) / 1000); + } + public static void main(String[] args) { - System.out.println("xdb maker"); + if (args.length < 1) { + printHelp(args); + return; + } + + try { + genDb(args); + } catch (Exception e) { + System.out.printf("failed running genDb: %s\n", e); + } } } diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java index d86f427..42108de 100644 --- a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java +++ b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java @@ -51,11 +51,12 @@ package org.lionsoul.ip2region.xdb; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.RandomAccessFile; +import javax.xml.crypto.Data; +import java.io.*; +import java.nio.charset.Charset; import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; import java.util.Map; public class Maker { @@ -66,11 +67,14 @@ public class Maker { public static final int VectorIndexCols = 256; public static final int VectorIndexSize = 8; public static final int SegmentIndexSize = 14; + public static final int VectorIndexLength = VectorIndexRows * VectorIndexCols * VectorIndexSize; private static final Log log = Log.getLogger(Maker.class); // source text file handle private final File srcFile; + private final List segments; + private final Charset bytesCharset; // destination binary file handle private final RandomAccessFile dstHandle; @@ -79,10 +83,10 @@ public class Maker { private final int indexPolicy; // region pool - private final Map regionPool; + private final Map regionPool; // vector index raw bytes - private byte[] vectorIndex; + private final byte[] vectorIndex; public Maker(int policy, String srcFile, String dstFile) throws FileNotFoundException { this.srcFile = new File(srcFile); @@ -90,9 +94,12 @@ public class Maker { throw new FileNotFoundException("source text file `" +srcFile+ "` not found"); } - this.dstHandle = new RandomAccessFile(dstFile, "r"); + this.bytesCharset = Charset.forName("utf-8"); + this.segments = new LinkedList(); + this.dstHandle = new RandomAccessFile(dstFile, "rw"); this.indexPolicy = policy; - this.regionPool = new HashMap(); + this.regionPool = new HashMap(); + this.vectorIndex = new byte[VectorIndexLength]; // all filled with 0 } // init the header of the target xdb binary file @@ -114,12 +121,53 @@ public class Maker { } // load all the segments - private void loadSegments() { + private void loadSegments() throws Exception { + log.infof("try to load the segments ... "); + long tStart = System.currentTimeMillis(); + Segment last = null; + String line; + final FileInputStream fis = new FileInputStream(srcFile); + final BufferedReader br = new BufferedReader(new InputStreamReader(fis, bytesCharset)); + while ((line = br.readLine()) != null) { + log.infof("load segment `%s`", line); + final String[] ps = line.split("\\|", 3); + if (ps.length != 3) { + br.close(); + throw new Exception("invalid ip segment line `"+ps[0]+"`"); + } + + long sip = Util.checkIP(ps[0]); + long eip = Util.checkIP(ps[1]); + if (sip > eip) { + br.close(); + throw new Exception("start ip("+ps[0]+") should not be greater than end ip("+ps[1]+")"); + } + + if (ps[2].length() < 1) { + br.close(); + throw new Exception("empty region info in segment line `"+ps[2]+"`"); + } + + // check the continuity of the data segment + if (last != null) { + if (last.endIP + 1 != sip) { + br.close(); + throw new Exception("discontinuous data segment: last.eip+1("+sip+") != seg.sip("+eip+", "+ps[0]+")"); + } + } + + Segment seg = new Segment(sip, eip, ps[2]); + segments.add(seg); + last = seg; + } + + br.close(); + log.infof("all segments loaded, length: %d, elapsed: %d ms", segments.size(), System.currentTimeMillis() - tStart); } // init the maker - public void init() throws IOException { + public void init() throws Exception { // init the db header initHeader(); @@ -127,9 +175,106 @@ public class Maker { loadSegments(); } - // start to make the binary file - public void make() { + // set the vector index info of the specified ip + private void setVectorIndex(long ip, long ptr) { + int il0 = (int) ((ip >> 24) & 0xFF); + int il1 = (int) ((ip >> 16) & 0xFF); + int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize; + long sPtr = Util.getIntLong(vectorIndex, idx); + if (sPtr == 0) { + Util.write(vectorIndex, idx, ptr, 4); + Util.write(vectorIndex, idx + 4, ptr, 4); + } else { + Util.write(vectorIndex, idx + 4, ptr, 4); + } + } + // start to make the binary file + public void start() throws Exception { + if (segments.size() == 0) { + throw new Exception("empty segment list"); + } + + // 1, write all the region/data to the binary file + dstHandle.seek(HeaderInfoLength + VectorIndexLength); + + log.infof("try to write the data block ... "); + for (Segment seg : segments) { + log.infof("try to write region `%s` ... ", seg.region); + DataEntry e = regionPool.get(seg.region); + if (e != null) { + log.infof(" --[Cached] with ptr=%d", e.ptr); + continue; + } + + // get the utf-8 bytes of the region info + byte[] regionBuff = seg.region.getBytes(bytesCharset); + if (regionBuff.length < 1) { + throw new Exception("empty region info for segment `"+seg+"`"); + } else if (regionBuff.length > 0xFFFF) { + throw new Exception("too long region info `"+seg.region+"`: should be less than 65535 bytes"); + } + + // record the current ptr + long pos = dstHandle.getFilePointer(); + dstHandle.write(regionBuff); + + // record the mapping + regionPool.put(seg.region, new DataEntry(regionBuff.length, pos)); + log.infof(" --[Added] with ptr=%d", pos); + } + + // 2, write the index block cache the super index block + log.infof("try to write the segment index block ... "); + int counter = 0; + long startIndexPtr = -1, endIndexPtr = -1; + byte[] indexBuff = new byte[SegmentIndexSize]; // 4 + 4 + 2 + 4 + for (Segment seg : segments) { + // we need the region ptr + DataEntry e = regionPool.get(seg.region); + if (e == null) { + throw new Exception("missing ptr cache for region `"+seg.region+"`"); + } + + List segList = seg.split(); + log.infof("try to index segment(%d splits) %s ... ", segList.size(), seg); + for (Segment s : segList) { + long pos = dstHandle.getFilePointer(); + + // encode the segment index info + Util.write(indexBuff, 0, s.startIP, 4); + Util.write(indexBuff, 4, s.endIP, 4); + Util.write(indexBuff, 8, e.length, 2); + Util.write(indexBuff, 10, e.ptr, 4); + dstHandle.write(indexBuff); + + log.infof("|-segment index: %d, ptr: %d, segment: %s", counter, pos, s); + setVectorIndex(s.startIP, pos); + counter++; + + // check and record the start index ptr + if (startIndexPtr == -1) { + startIndexPtr = pos; + } + + endIndexPtr = pos; + } + } + + // 3, synchronize the vector index block + log.infof("try to write the vector index block ... "); + dstHandle.seek(HeaderInfoLength); + dstHandle.write(vectorIndex); + + // 4, synchronize the segment index info + log.infof("try to write the segment index ptr ... "); + Util.write(indexBuff, 0, startIndexPtr, 4); + Util.write(indexBuff, 4, endIndexPtr, 4); + dstHandle.seek(8); + dstHandle.write(indexBuff, 0, 8); + + log.infof("write done, dataBlocks: %d, indexBlocks: (%d, %d), indexPtr: (%d, %d)", + regionPool.size(), segments.size(), counter, startIndexPtr, endIndexPtr); } // end the make, do the resource clean up @@ -137,4 +282,14 @@ public class Maker { this.dstHandle.close(); } + private static class DataEntry { + long ptr; + int length; // in bytes + + DataEntry(int length, long ptr) { + this.length = length; + this.ptr = ptr; + } + } + } From 58d9daa85b211e9563d6bc8d4e1c815bf4c977d0 Mon Sep 17 00:00:00 2001 From: lion Date: Fri, 15 Jul 2022 20:26:47 +0800 Subject: [PATCH 7/7] Add ReadMe Doc --- maker/java/ReadMe.md | 33 +++++++++++++++++-- .../org/lionsoul/ip2region/MakerTest.java | 2 +- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/maker/java/ReadMe.md b/maker/java/ReadMe.md index ed086ac..f11959d 100644 --- a/maker/java/ReadMe.md +++ b/maker/java/ReadMe.md @@ -1,7 +1,36 @@ # ip2region xdb java 生成实现 +# 编译安装 +通过 maven 来编译可运行 jar 程序： +```bash +# cd 到 maker/java 根目录 +mvn clean compile package +``` + +然会会在当前目录的 target 目录下得到一个 ip2region-maker-{version}.jar 的打包文件。 + # 数据生成 -# 数据查询 +通过 `java -jar ip2region-maker-{version}.jar` 来生成 ip2region.xdb 二进制文件： +```bash +➜ java git:(java_xdb_maker) ✗ java -jar ./target/ip2region-maker-1.0.0.jar +ip2region xdb maker +java -jar ip2region-maker-{version}.jar [command options] +options: + --src string source ip text file path + --dst string destination binary xdb file path +``` -# bench 测试 +例如，通过默认的 data/ip.merge.txt 原数据，在当前目录生成一个 ip2region.xdb 二进制文件： +```bash +# 会看到一堆输出，最终会看到如下输出表示运行成功 +... +2022-07-15 20:21:29 INFO org.lionsoul.ip2region.xdb.Maker try to write the vector index block ... +2022-07-15 20:21:29 INFO org.lionsoul.ip2region.xdb.Maker try to write the segment index ptr ... +2022-07-15 20:21:29 INFO org.lionsoul.ip2region.xdb.Maker write done, dataBlocks: 13804, indexBlocks: (683591, 720221), indexPtr: (982904, 11065984) +2022-07-15 20:21:29 INFO org.lionsoul.ip2region.MakerTest Done, elapsed: 50 s +``` + +# 数据 查询/bench 测试 + +已经完成开发的 [binding](../../binding/) 都有查询和 bench 测试程序以及使用文档，你可以使用你熟悉的语言的 searcher 进行查询测试或者bench测试，来确认数据的正确性和完整性。 diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java b/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java index 414a2da..6feca11 100644 --- a/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java +++ b/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java @@ -71,7 +71,7 @@ public class MakerTest { maker.start(); maker.end(); - log.infof("Done, elapsed: %d s\n", (System.currentTimeMillis() - tStart) / 1000); + log.infof("Done, elapsed: %d s", (System.currentTimeMillis() - tStart) / 1000); } public static void main(String[] args) {