maker and maker test implemented
This commit is contained in:
Lion
parent
8f421f554d
commit
061f2f8c91
|
|
@ -3,8 +3,8 @@
|
|||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.lionsoul</groupId>
|
||||
<artifactId>ip2region</artifactId>
|
||||
<version>2.6.4</version>
|
||||
<artifactId>ip2region-maker</artifactId>
|
||||
<version>1.0.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>ip2region</name>
|
||||
|
|
@ -47,7 +47,7 @@
|
|||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.13.1</version>
|
||||
<version>4.13.2</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
|
@ -110,6 +110,14 @@
|
|||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<source>6</source>
|
||||
<target>6</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
|
|
|||
|
|
@ -7,10 +7,84 @@
|
|||
|
||||
package org.lionsoul.ip2region;
|
||||
|
||||
import org.lionsoul.ip2region.xdb.IndexPolicy;
|
||||
import org.lionsoul.ip2region.xdb.Log;
|
||||
import org.lionsoul.ip2region.xdb.Maker;
|
||||
|
||||
public class MakerTest {
|
||||
|
||||
public final static Log log = Log.getLogger(MakerTest.class);
|
||||
|
||||
public static void printHelp(String[] args) {
|
||||
System.out.println("ip2region xdb maker");
|
||||
System.out.println("java -jar ip2region-maker-{version}.jar [command options]");
|
||||
System.out.println("options:");
|
||||
System.out.println(" --src string source ip text file path");
|
||||
System.out.println(" --dst string destination binary xdb file path");
|
||||
}
|
||||
|
||||
public static void genDb(String[] args) throws Exception {
|
||||
String srcFile = "", dstFile = "";
|
||||
int indexPolicy = IndexPolicy.Vector;
|
||||
for (final String r : args) {
|
||||
if (r.length() < 5) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (r.indexOf("--") != 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int sIdx = r.indexOf('=');
|
||||
if (sIdx < 0) {
|
||||
System.out.printf("missing = for args pair `%s`\n", r);
|
||||
return;
|
||||
}
|
||||
|
||||
String key = r.substring(2, sIdx);
|
||||
String val = r.substring(sIdx + 1);
|
||||
// System.out.printf("key=%s, val=%s\n", key, val);
|
||||
if ("src".equals(key)) {
|
||||
srcFile = val;
|
||||
} else if ("dst".equals(key)) {
|
||||
dstFile = val;
|
||||
} else if ("index".equals(key)) {
|
||||
try {
|
||||
indexPolicy = IndexPolicy.parse(val);
|
||||
} catch (Exception e) {
|
||||
System.out.println("parse policy " + e);
|
||||
}
|
||||
} else {
|
||||
System.out.printf("undefined option `%s`\n", r);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (srcFile.length() < 1 || dstFile.length() < 1) {
|
||||
printHelp(args);
|
||||
return;
|
||||
}
|
||||
|
||||
long tStart = System.currentTimeMillis();
|
||||
Maker maker = new Maker(indexPolicy, srcFile, dstFile);
|
||||
maker.init();
|
||||
maker.start();
|
||||
maker.end();
|
||||
|
||||
log.infof("Done, elapsed: %d s\n", (System.currentTimeMillis() - tStart) / 1000);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println("xdb maker");
|
||||
if (args.length < 1) {
|
||||
printHelp(args);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
genDb(args);
|
||||
} catch (Exception e) {
|
||||
System.out.printf("failed running genDb: %s\n", e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -51,11 +51,12 @@
|
|||
|
||||
package org.lionsoul.ip2region.xdb;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.RandomAccessFile;
|
||||
import javax.xml.crypto.Data;
|
||||
import java.io.*;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class Maker {
|
||||
|
|
@ -66,11 +67,14 @@ public class Maker {
|
|||
public static final int VectorIndexCols = 256;
|
||||
public static final int VectorIndexSize = 8;
|
||||
public static final int SegmentIndexSize = 14;
|
||||
public static final int VectorIndexLength = VectorIndexRows * VectorIndexCols * VectorIndexSize;
|
||||
|
||||
private static final Log log = Log.getLogger(Maker.class);
|
||||
|
||||
// source text file handle
|
||||
private final File srcFile;
|
||||
private final List<Segment> segments;
|
||||
private final Charset bytesCharset;
|
||||
|
||||
// destination binary file handle
|
||||
private final RandomAccessFile dstHandle;
|
||||
|
|
@ -79,10 +83,10 @@ public class Maker {
|
|||
private final int indexPolicy;
|
||||
|
||||
// region pool
|
||||
private final Map<String, Long> regionPool;
|
||||
private final Map<String, DataEntry> regionPool;
|
||||
|
||||
// vector index raw bytes
|
||||
private byte[] vectorIndex;
|
||||
private final byte[] vectorIndex;
|
||||
|
||||
public Maker(int policy, String srcFile, String dstFile) throws FileNotFoundException {
|
||||
this.srcFile = new File(srcFile);
|
||||
|
|
@ -90,9 +94,12 @@ public class Maker {
|
|||
throw new FileNotFoundException("source text file `" +srcFile+ "` not found");
|
||||
}
|
||||
|
||||
this.dstHandle = new RandomAccessFile(dstFile, "r");
|
||||
this.bytesCharset = Charset.forName("utf-8");
|
||||
this.segments = new LinkedList<Segment>();
|
||||
this.dstHandle = new RandomAccessFile(dstFile, "rw");
|
||||
this.indexPolicy = policy;
|
||||
this.regionPool = new HashMap<String, Long>();
|
||||
this.regionPool = new HashMap<String, DataEntry>();
|
||||
this.vectorIndex = new byte[VectorIndexLength]; // all filled with 0
|
||||
}
|
||||
|
||||
// init the header of the target xdb binary file
|
||||
|
|
@ -114,12 +121,53 @@ public class Maker {
|
|||
}
|
||||
|
||||
// load all the segments
|
||||
private void loadSegments() {
|
||||
private void loadSegments() throws Exception {
|
||||
log.infof("try to load the segments ... ");
|
||||
long tStart = System.currentTimeMillis();
|
||||
Segment last = null;
|
||||
String line;
|
||||
|
||||
final FileInputStream fis = new FileInputStream(srcFile);
|
||||
final BufferedReader br = new BufferedReader(new InputStreamReader(fis, bytesCharset));
|
||||
while ((line = br.readLine()) != null) {
|
||||
log.infof("load segment `%s`", line);
|
||||
final String[] ps = line.split("\\|", 3);
|
||||
if (ps.length != 3) {
|
||||
br.close();
|
||||
throw new Exception("invalid ip segment line `"+ps[0]+"`");
|
||||
}
|
||||
|
||||
long sip = Util.checkIP(ps[0]);
|
||||
long eip = Util.checkIP(ps[1]);
|
||||
if (sip > eip) {
|
||||
br.close();
|
||||
throw new Exception("start ip("+ps[0]+") should not be greater than end ip("+ps[1]+")");
|
||||
}
|
||||
|
||||
if (ps[2].length() < 1) {
|
||||
br.close();
|
||||
throw new Exception("empty region info in segment line `"+ps[2]+"`");
|
||||
}
|
||||
|
||||
// check the continuity of the data segment
|
||||
if (last != null) {
|
||||
if (last.endIP + 1 != sip) {
|
||||
br.close();
|
||||
throw new Exception("discontinuous data segment: last.eip+1("+sip+") != seg.sip("+eip+", "+ps[0]+")");
|
||||
}
|
||||
}
|
||||
|
||||
Segment seg = new Segment(sip, eip, ps[2]);
|
||||
segments.add(seg);
|
||||
last = seg;
|
||||
}
|
||||
|
||||
br.close();
|
||||
log.infof("all segments loaded, length: %d, elapsed: %d ms", segments.size(), System.currentTimeMillis() - tStart);
|
||||
}
|
||||
|
||||
// init the maker
|
||||
public void init() throws IOException {
|
||||
public void init() throws Exception {
|
||||
// init the db header
|
||||
initHeader();
|
||||
|
||||
|
|
@ -127,9 +175,106 @@ public class Maker {
|
|||
loadSegments();
|
||||
}
|
||||
|
||||
// start to make the binary file
|
||||
public void make() {
|
||||
// set the vector index info of the specified ip
|
||||
private void setVectorIndex(long ip, long ptr) {
|
||||
int il0 = (int) ((ip >> 24) & 0xFF);
|
||||
int il1 = (int) ((ip >> 16) & 0xFF);
|
||||
int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
|
||||
long sPtr = Util.getIntLong(vectorIndex, idx);
|
||||
if (sPtr == 0) {
|
||||
Util.write(vectorIndex, idx, ptr, 4);
|
||||
Util.write(vectorIndex, idx + 4, ptr, 4);
|
||||
} else {
|
||||
Util.write(vectorIndex, idx + 4, ptr, 4);
|
||||
}
|
||||
}
|
||||
|
||||
// start to make the binary file
|
||||
public void start() throws Exception {
|
||||
if (segments.size() == 0) {
|
||||
throw new Exception("empty segment list");
|
||||
}
|
||||
|
||||
// 1, write all the region/data to the binary file
|
||||
dstHandle.seek(HeaderInfoLength + VectorIndexLength);
|
||||
|
||||
log.infof("try to write the data block ... ");
|
||||
for (Segment seg : segments) {
|
||||
log.infof("try to write region `%s` ... ", seg.region);
|
||||
DataEntry e = regionPool.get(seg.region);
|
||||
if (e != null) {
|
||||
log.infof(" --[Cached] with ptr=%d", e.ptr);
|
||||
continue;
|
||||
}
|
||||
|
||||
// get the utf-8 bytes of the region info
|
||||
byte[] regionBuff = seg.region.getBytes(bytesCharset);
|
||||
if (regionBuff.length < 1) {
|
||||
throw new Exception("empty region info for segment `"+seg+"`");
|
||||
} else if (regionBuff.length > 0xFFFF) {
|
||||
throw new Exception("too long region info `"+seg.region+"`: should be less than 65535 bytes");
|
||||
}
|
||||
|
||||
// record the current ptr
|
||||
long pos = dstHandle.getFilePointer();
|
||||
dstHandle.write(regionBuff);
|
||||
|
||||
// record the mapping
|
||||
regionPool.put(seg.region, new DataEntry(regionBuff.length, pos));
|
||||
log.infof(" --[Added] with ptr=%d", pos);
|
||||
}
|
||||
|
||||
// 2, write the index block cache the super index block
|
||||
log.infof("try to write the segment index block ... ");
|
||||
int counter = 0;
|
||||
long startIndexPtr = -1, endIndexPtr = -1;
|
||||
byte[] indexBuff = new byte[SegmentIndexSize]; // 4 + 4 + 2 + 4
|
||||
for (Segment seg : segments) {
|
||||
// we need the region ptr
|
||||
DataEntry e = regionPool.get(seg.region);
|
||||
if (e == null) {
|
||||
throw new Exception("missing ptr cache for region `"+seg.region+"`");
|
||||
}
|
||||
|
||||
List<Segment> segList = seg.split();
|
||||
log.infof("try to index segment(%d splits) %s ... ", segList.size(), seg);
|
||||
for (Segment s : segList) {
|
||||
long pos = dstHandle.getFilePointer();
|
||||
|
||||
// encode the segment index info
|
||||
Util.write(indexBuff, 0, s.startIP, 4);
|
||||
Util.write(indexBuff, 4, s.endIP, 4);
|
||||
Util.write(indexBuff, 8, e.length, 2);
|
||||
Util.write(indexBuff, 10, e.ptr, 4);
|
||||
dstHandle.write(indexBuff);
|
||||
|
||||
log.infof("|-segment index: %d, ptr: %d, segment: %s", counter, pos, s);
|
||||
setVectorIndex(s.startIP, pos);
|
||||
counter++;
|
||||
|
||||
// check and record the start index ptr
|
||||
if (startIndexPtr == -1) {
|
||||
startIndexPtr = pos;
|
||||
}
|
||||
|
||||
endIndexPtr = pos;
|
||||
}
|
||||
}
|
||||
|
||||
// 3, synchronize the vector index block
|
||||
log.infof("try to write the vector index block ... ");
|
||||
dstHandle.seek(HeaderInfoLength);
|
||||
dstHandle.write(vectorIndex);
|
||||
|
||||
// 4, synchronize the segment index info
|
||||
log.infof("try to write the segment index ptr ... ");
|
||||
Util.write(indexBuff, 0, startIndexPtr, 4);
|
||||
Util.write(indexBuff, 4, endIndexPtr, 4);
|
||||
dstHandle.seek(8);
|
||||
dstHandle.write(indexBuff, 0, 8);
|
||||
|
||||
log.infof("write done, dataBlocks: %d, indexBlocks: (%d, %d), indexPtr: (%d, %d)",
|
||||
regionPool.size(), segments.size(), counter, startIndexPtr, endIndexPtr);
|
||||
}
|
||||
|
||||
// end the make, do the resource clean up
|
||||
|
|
@ -137,4 +282,14 @@ public class Maker {
|
|||
this.dstHandle.close();
|
||||
}
|
||||
|
||||
private static class DataEntry {
|
||||
long ptr;
|
||||
int length; // in bytes
|
||||
|
||||
DataEntry(int length, long ptr) {
|
||||
this.length = length;
|
||||
this.ptr = ptr;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue