xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

maker and maker test implemented

This commit is contained in:
Lion 2022-07-15 19:05:42 +08:00
parent 01529b8887
commit 4d3d5501b5
3 changed files with 253 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
maker/java/pom.xml
View File

@ -3,8 +3,8 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.lionsoul</groupId>
<artifactId>ip2region</artifactId>
<version>2.6.4</version>
<artifactId>ip2region-maker</artifactId>
<version>1.0.0</version>
<packaging>jar</packaging>
<name>ip2region</name>
@ -47,7 +47,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.1</version>
<version>4.13.2</version>
<scope>test</scope>
</dependency>
</dependencies>
@ -110,6 +110,14 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>6</source>
<target>6</target>
</configuration>
</plugin>
</plugins>
</build>

76
maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java
View File

@ -7,10 +7,84 @@
package org.lionsoul.ip2region;
import org.lionsoul.ip2region.xdb.IndexPolicy;
import org.lionsoul.ip2region.xdb.Log;
import org.lionsoul.ip2region.xdb.Maker;
public class MakerTest {
public final static Log log = Log.getLogger(MakerTest.class);
public static void printHelp(String[] args) {
System.out.println("ip2region xdb maker");
System.out.println("java -jar ip2region-maker-{version}.jar [command options]");
System.out.println("options:");
System.out.println(" --src string source ip text file path");
System.out.println(" --dst string destination binary xdb file path");
}
public static void genDb(String[] args) throws Exception {
String srcFile = "", dstFile = "";
int indexPolicy = IndexPolicy.Vector;
for (final String r : args) {
if (r.length() < 5) {
continue;
}
if (r.indexOf("--") != 0) {
continue;
}
int sIdx = r.indexOf('=');
if (sIdx < 0) {
System.out.printf("missing = for args pair `%s`\n", r);
return;
}
String key = r.substring(2, sIdx);
String val = r.substring(sIdx + 1);
// System.out.printf("key=%s, val=%s\n", key, val);
if ("src".equals(key)) {
srcFile = val;
} else if ("dst".equals(key)) {
dstFile = val;
} else if ("index".equals(key)) {
try {
indexPolicy = IndexPolicy.parse(val);
} catch (Exception e) {
System.out.println("parse policy " + e);
}
} else {
System.out.printf("undefined option `%s`\n", r);
return;
}
}
if (srcFile.length() < 1 || dstFile.length() < 1) {
printHelp(args);
return;
}
long tStart = System.currentTimeMillis();
Maker maker = new Maker(indexPolicy, srcFile, dstFile);
maker.init();
maker.start();
maker.end();
log.infof("Done, elapsed: %d s\n", (System.currentTimeMillis() - tStart) / 1000);
}
public static void main(String[] args) {
System.out.println("xdb maker");
if (args.length < 1) {
printHelp(args);
return;
}
try {
genDb(args);
} catch (Exception e) {
System.out.printf("failed running genDb: %s\n", e);
}
}
}

179
maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java
View File

@ -51,11 +51,12 @@
package org.lionsoul.ip2region.xdb;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import javax.xml.crypto.Data;
import java.io.*;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
public class Maker {
@ -66,11 +67,14 @@ public class Maker {
public static final int VectorIndexCols = 256;
public static final int VectorIndexSize = 8;
public static final int SegmentIndexSize = 14;
public static final int VectorIndexLength = VectorIndexRows * VectorIndexCols * VectorIndexSize;
private static final Log log = Log.getLogger(Maker.class);
// source text file handle
private final File srcFile;
private final List<Segment> segments;
private final Charset bytesCharset;
// destination binary file handle
private final RandomAccessFile dstHandle;
@ -79,10 +83,10 @@ public class Maker {
private final int indexPolicy;
// region pool
private final Map<String, Long> regionPool;
private final Map<String, DataEntry> regionPool;
// vector index raw bytes
private byte[] vectorIndex;
private final byte[] vectorIndex;
public Maker(int policy, String srcFile, String dstFile) throws FileNotFoundException {
this.srcFile = new File(srcFile);
@ -90,9 +94,12 @@ public class Maker {
throw new FileNotFoundException("source text file `" +srcFile+ "` not found");
}
this.dstHandle = new RandomAccessFile(dstFile, "r");
this.bytesCharset = Charset.forName("utf-8");
this.segments = new LinkedList<Segment>();
this.dstHandle = new RandomAccessFile(dstFile, "rw");
this.indexPolicy = policy;
this.regionPool = new HashMap<String, Long>();
this.regionPool = new HashMap<String, DataEntry>();
this.vectorIndex = new byte[VectorIndexLength]; // all filled with 0
}
// init the header of the target xdb binary file
@ -114,12 +121,53 @@ public class Maker {
}
// load all the segments
private void loadSegments() {
private void loadSegments() throws Exception {
log.infof("try to load the segments ... ");
long tStart = System.currentTimeMillis();
Segment last = null;
String line;
final FileInputStream fis = new FileInputStream(srcFile);
final BufferedReader br = new BufferedReader(new InputStreamReader(fis, bytesCharset));
while ((line = br.readLine()) != null) {
log.infof("load segment `%s`", line);
final String[] ps = line.split("\\|", 3);
if (ps.length != 3) {
br.close();
throw new Exception("invalid ip segment line `"+ps[0]+"`");
}
long sip = Util.checkIP(ps[0]);
long eip = Util.checkIP(ps[1]);
if (sip > eip) {
br.close();
throw new Exception("start ip("+ps[0]+") should not be greater than end ip("+ps[1]+")");
}
if (ps[2].length() < 1) {
br.close();
throw new Exception("empty region info in segment line `"+ps[2]+"`");
}
// check the continuity of the data segment
if (last != null) {
if (last.endIP + 1 != sip) {
br.close();
throw new Exception("discontinuous data segment: last.eip+1("+sip+") != seg.sip("+eip+", "+ps[0]+")");
}
}
Segment seg = new Segment(sip, eip, ps[2]);
segments.add(seg);
last = seg;
}
br.close();
log.infof("all segments loaded, length: %d, elapsed: %d ms", segments.size(), System.currentTimeMillis() - tStart);
}
// init the maker
public void init() throws IOException {
public void init() throws Exception {
// init the db header
initHeader();
@ -127,9 +175,106 @@ public class Maker {
loadSegments();
}
// start to make the binary file
public void make() {
// set the vector index info of the specified ip
private void setVectorIndex(long ip, long ptr) {
int il0 = (int) ((ip >> 24) & 0xFF);
int il1 = (int) ((ip >> 16) & 0xFF);
int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
long sPtr = Util.getIntLong(vectorIndex, idx);
if (sPtr == 0) {
Util.write(vectorIndex, idx, ptr, 4);
Util.write(vectorIndex, idx + 4, ptr, 4);
} else {
Util.write(vectorIndex, idx + 4, ptr, 4);
}
}
// start to make the binary file
public void start() throws Exception {
if (segments.size() == 0) {
throw new Exception("empty segment list");
}
// 1, write all the region/data to the binary file
dstHandle.seek(HeaderInfoLength + VectorIndexLength);
log.infof("try to write the data block ... ");
for (Segment seg : segments) {
log.infof("try to write region `%s` ... ", seg.region);
DataEntry e = regionPool.get(seg.region);
if (e != null) {
log.infof(" --[Cached] with ptr=%d", e.ptr);
continue;
}
// get the utf-8 bytes of the region info
byte[] regionBuff = seg.region.getBytes(bytesCharset);
if (regionBuff.length < 1) {
throw new Exception("empty region info for segment `"+seg+"`");
} else if (regionBuff.length > 0xFFFF) {
throw new Exception("too long region info `"+seg.region+"`: should be less than 65535 bytes");
}
// record the current ptr
long pos = dstHandle.getFilePointer();
dstHandle.write(regionBuff);
// record the mapping
regionPool.put(seg.region, new DataEntry(regionBuff.length, pos));
log.infof(" --[Added] with ptr=%d", pos);
}
// 2, write the index block cache the super index block
log.infof("try to write the segment index block ... ");
int counter = 0;
long startIndexPtr = -1, endIndexPtr = -1;
byte[] indexBuff = new byte[SegmentIndexSize]; // 4 + 4 + 2 + 4
for (Segment seg : segments) {
// we need the region ptr
DataEntry e = regionPool.get(seg.region);
if (e == null) {
throw new Exception("missing ptr cache for region `"+seg.region+"`");
}
List<Segment> segList = seg.split();
log.infof("try to index segment(%d splits) %s ... ", segList.size(), seg);
for (Segment s : segList) {
long pos = dstHandle.getFilePointer();
// encode the segment index info
Util.write(indexBuff, 0, s.startIP, 4);
Util.write(indexBuff, 4, s.endIP, 4);
Util.write(indexBuff, 8, e.length, 2);
Util.write(indexBuff, 10, e.ptr, 4);
dstHandle.write(indexBuff);
log.infof("|-segment index: %d, ptr: %d, segment: %s", counter, pos, s);
setVectorIndex(s.startIP, pos);
counter++;
// check and record the start index ptr
if (startIndexPtr == -1) {
startIndexPtr = pos;
}
endIndexPtr = pos;
}
}
// 3, synchronize the vector index block
log.infof("try to write the vector index block ... ");
dstHandle.seek(HeaderInfoLength);
dstHandle.write(vectorIndex);
// 4, synchronize the segment index info
log.infof("try to write the segment index ptr ... ");
Util.write(indexBuff, 0, startIndexPtr, 4);
Util.write(indexBuff, 4, endIndexPtr, 4);
dstHandle.seek(8);
dstHandle.write(indexBuff, 0, 8);
log.infof("write done, dataBlocks: %d, indexBlocks: (%d, %d), indexPtr: (%d, %d)",
regionPool.size(), segments.size(), counter, startIndexPtr, endIndexPtr);
}
// end the make, do the resource clean up
@ -137,4 +282,14 @@ public class Maker {
this.dstHandle.close();
}
private static class DataEntry {
long ptr;
int length; // in bytes
DataEntry(int length, long ptr) {
this.length = length;
this.ptr = ptr;
}
}
}