diff --git a/maker/java/pom.xml b/maker/java/pom.xml
index 391aef7..1bf6653 100644
--- a/maker/java/pom.xml
+++ b/maker/java/pom.xml
@@ -3,8 +3,8 @@
4.0.0
org.lionsoul
- ip2region
- 2.6.4
+ ip2region-maker
+ 1.0.0
jar
ip2region
@@ -47,7 +47,7 @@
junit
junit
- 4.13.1
+ 4.13.2
test
@@ -110,6 +110,14 @@
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+
+ 6
+ 6
+
+
diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java b/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java
index 017facd..414a2da 100644
--- a/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java
+++ b/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java
@@ -7,10 +7,84 @@
package org.lionsoul.ip2region;
+import org.lionsoul.ip2region.xdb.IndexPolicy;
+import org.lionsoul.ip2region.xdb.Log;
+import org.lionsoul.ip2region.xdb.Maker;
+
public class MakerTest {
+ public final static Log log = Log.getLogger(MakerTest.class);
+
+ public static void printHelp(String[] args) {
+ System.out.println("ip2region xdb maker");
+ System.out.println("java -jar ip2region-maker-{version}.jar [command options]");
+ System.out.println("options:");
+ System.out.println(" --src string source ip text file path");
+ System.out.println(" --dst string destination binary xdb file path");
+ }
+
+ public static void genDb(String[] args) throws Exception {
+ String srcFile = "", dstFile = "";
+ int indexPolicy = IndexPolicy.Vector;
+ for (final String r : args) {
+ if (r.length() < 5) {
+ continue;
+ }
+
+ if (r.indexOf("--") != 0) {
+ continue;
+ }
+
+ int sIdx = r.indexOf('=');
+ if (sIdx < 0) {
+ System.out.printf("missing = for args pair `%s`\n", r);
+ return;
+ }
+
+ String key = r.substring(2, sIdx);
+ String val = r.substring(sIdx + 1);
+ // System.out.printf("key=%s, val=%s\n", key, val);
+ if ("src".equals(key)) {
+ srcFile = val;
+ } else if ("dst".equals(key)) {
+ dstFile = val;
+ } else if ("index".equals(key)) {
+ try {
+ indexPolicy = IndexPolicy.parse(val);
+ } catch (Exception e) {
+ System.out.println("parse policy " + e);
+ }
+ } else {
+ System.out.printf("undefined option `%s`\n", r);
+ return;
+ }
+ }
+
+ if (srcFile.length() < 1 || dstFile.length() < 1) {
+ printHelp(args);
+ return;
+ }
+
+ long tStart = System.currentTimeMillis();
+ Maker maker = new Maker(indexPolicy, srcFile, dstFile);
+ maker.init();
+ maker.start();
+ maker.end();
+
+ log.infof("Done, elapsed: %d s\n", (System.currentTimeMillis() - tStart) / 1000);
+ }
+
public static void main(String[] args) {
- System.out.println("xdb maker");
+ if (args.length < 1) {
+ printHelp(args);
+ return;
+ }
+
+ try {
+ genDb(args);
+ } catch (Exception e) {
+ System.out.printf("failed running genDb: %s\n", e);
+ }
}
}
diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java
index d86f427..42108de 100644
--- a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java
+++ b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java
@@ -51,11 +51,12 @@
package org.lionsoul.ip2region.xdb;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.RandomAccessFile;
+import javax.xml.crypto.Data;
+import java.io.*;
+import java.nio.charset.Charset;
import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
public class Maker {
@@ -66,11 +67,14 @@ public class Maker {
public static final int VectorIndexCols = 256;
public static final int VectorIndexSize = 8;
public static final int SegmentIndexSize = 14;
+ public static final int VectorIndexLength = VectorIndexRows * VectorIndexCols * VectorIndexSize;
private static final Log log = Log.getLogger(Maker.class);
// source text file handle
private final File srcFile;
+ private final List segments;
+ private final Charset bytesCharset;
// destination binary file handle
private final RandomAccessFile dstHandle;
@@ -79,10 +83,10 @@ public class Maker {
private final int indexPolicy;
// region pool
- private final Map regionPool;
+ private final Map regionPool;
// vector index raw bytes
- private byte[] vectorIndex;
+ private final byte[] vectorIndex;
public Maker(int policy, String srcFile, String dstFile) throws FileNotFoundException {
this.srcFile = new File(srcFile);
@@ -90,9 +94,12 @@ public class Maker {
throw new FileNotFoundException("source text file `" +srcFile+ "` not found");
}
- this.dstHandle = new RandomAccessFile(dstFile, "r");
+ this.bytesCharset = Charset.forName("utf-8");
+ this.segments = new LinkedList();
+ this.dstHandle = new RandomAccessFile(dstFile, "rw");
this.indexPolicy = policy;
- this.regionPool = new HashMap();
+ this.regionPool = new HashMap();
+ this.vectorIndex = new byte[VectorIndexLength]; // all filled with 0
}
// init the header of the target xdb binary file
@@ -114,12 +121,53 @@ public class Maker {
}
// load all the segments
- private void loadSegments() {
+ private void loadSegments() throws Exception {
+ log.infof("try to load the segments ... ");
+ long tStart = System.currentTimeMillis();
+ Segment last = null;
+ String line;
+ final FileInputStream fis = new FileInputStream(srcFile);
+ final BufferedReader br = new BufferedReader(new InputStreamReader(fis, bytesCharset));
+ while ((line = br.readLine()) != null) {
+ log.infof("load segment `%s`", line);
+ final String[] ps = line.split("\\|", 3);
+ if (ps.length != 3) {
+ br.close();
+ throw new Exception("invalid ip segment line `"+ps[0]+"`");
+ }
+
+ long sip = Util.checkIP(ps[0]);
+ long eip = Util.checkIP(ps[1]);
+ if (sip > eip) {
+ br.close();
+ throw new Exception("start ip("+ps[0]+") should not be greater than end ip("+ps[1]+")");
+ }
+
+ if (ps[2].length() < 1) {
+ br.close();
+ throw new Exception("empty region info in segment line `"+ps[2]+"`");
+ }
+
+ // check the continuity of the data segment
+ if (last != null) {
+ if (last.endIP + 1 != sip) {
+ br.close();
+ throw new Exception("discontinuous data segment: last.eip+1("+sip+") != seg.sip("+eip+", "+ps[0]+")");
+ }
+ }
+
+ Segment seg = new Segment(sip, eip, ps[2]);
+ segments.add(seg);
+ last = seg;
+ }
+
+ br.close();
+ log.infof("all segments loaded, length: %d, elapsed: %d ms", segments.size(), System.currentTimeMillis() - tStart);
}
// init the maker
- public void init() throws IOException {
+ public void init() throws Exception {
// init the db header
initHeader();
@@ -127,9 +175,106 @@ public class Maker {
loadSegments();
}
- // start to make the binary file
- public void make() {
+ // set the vector index info of the specified ip
+ private void setVectorIndex(long ip, long ptr) {
+ int il0 = (int) ((ip >> 24) & 0xFF);
+ int il1 = (int) ((ip >> 16) & 0xFF);
+ int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
+ long sPtr = Util.getIntLong(vectorIndex, idx);
+ if (sPtr == 0) {
+ Util.write(vectorIndex, idx, ptr, 4);
+ Util.write(vectorIndex, idx + 4, ptr, 4);
+ } else {
+ Util.write(vectorIndex, idx + 4, ptr, 4);
+ }
+ }
+ // start to make the binary file
+ public void start() throws Exception {
+ if (segments.size() == 0) {
+ throw new Exception("empty segment list");
+ }
+
+ // 1, write all the region/data to the binary file
+ dstHandle.seek(HeaderInfoLength + VectorIndexLength);
+
+ log.infof("try to write the data block ... ");
+ for (Segment seg : segments) {
+ log.infof("try to write region `%s` ... ", seg.region);
+ DataEntry e = regionPool.get(seg.region);
+ if (e != null) {
+ log.infof(" --[Cached] with ptr=%d", e.ptr);
+ continue;
+ }
+
+ // get the utf-8 bytes of the region info
+ byte[] regionBuff = seg.region.getBytes(bytesCharset);
+ if (regionBuff.length < 1) {
+ throw new Exception("empty region info for segment `"+seg+"`");
+ } else if (regionBuff.length > 0xFFFF) {
+ throw new Exception("too long region info `"+seg.region+"`: should be less than 65535 bytes");
+ }
+
+ // record the current ptr
+ long pos = dstHandle.getFilePointer();
+ dstHandle.write(regionBuff);
+
+ // record the mapping
+ regionPool.put(seg.region, new DataEntry(regionBuff.length, pos));
+ log.infof(" --[Added] with ptr=%d", pos);
+ }
+
+ // 2, write the index block cache the super index block
+ log.infof("try to write the segment index block ... ");
+ int counter = 0;
+ long startIndexPtr = -1, endIndexPtr = -1;
+ byte[] indexBuff = new byte[SegmentIndexSize]; // 4 + 4 + 2 + 4
+ for (Segment seg : segments) {
+ // we need the region ptr
+ DataEntry e = regionPool.get(seg.region);
+ if (e == null) {
+ throw new Exception("missing ptr cache for region `"+seg.region+"`");
+ }
+
+ List segList = seg.split();
+ log.infof("try to index segment(%d splits) %s ... ", segList.size(), seg);
+ for (Segment s : segList) {
+ long pos = dstHandle.getFilePointer();
+
+ // encode the segment index info
+ Util.write(indexBuff, 0, s.startIP, 4);
+ Util.write(indexBuff, 4, s.endIP, 4);
+ Util.write(indexBuff, 8, e.length, 2);
+ Util.write(indexBuff, 10, e.ptr, 4);
+ dstHandle.write(indexBuff);
+
+ log.infof("|-segment index: %d, ptr: %d, segment: %s", counter, pos, s);
+ setVectorIndex(s.startIP, pos);
+ counter++;
+
+ // check and record the start index ptr
+ if (startIndexPtr == -1) {
+ startIndexPtr = pos;
+ }
+
+ endIndexPtr = pos;
+ }
+ }
+
+ // 3, synchronize the vector index block
+ log.infof("try to write the vector index block ... ");
+ dstHandle.seek(HeaderInfoLength);
+ dstHandle.write(vectorIndex);
+
+ // 4, synchronize the segment index info
+ log.infof("try to write the segment index ptr ... ");
+ Util.write(indexBuff, 0, startIndexPtr, 4);
+ Util.write(indexBuff, 4, endIndexPtr, 4);
+ dstHandle.seek(8);
+ dstHandle.write(indexBuff, 0, 8);
+
+ log.infof("write done, dataBlocks: %d, indexBlocks: (%d, %d), indexPtr: (%d, %d)",
+ regionPool.size(), segments.size(), counter, startIndexPtr, endIndexPtr);
}
// end the make, do the resource clean up
@@ -137,4 +282,14 @@ public class Maker {
this.dstHandle.close();
}
+ private static class DataEntry {
+ long ptr;
+ int length; // in bytes
+
+ DataEntry(int length, long ptr) {
+ this.length = length;
+ this.ptr = ptr;
+ }
+ }
+
}