use the final
This commit is contained in:
parent
2c46a386cc
commit
0c960c4122
|
|
@ -87,18 +87,27 @@ public class Maker {
|
||||||
// vector index raw bytes
|
// vector index raw bytes
|
||||||
private final byte[] vectorIndex;
|
private final byte[] vectorIndex;
|
||||||
|
|
||||||
public Maker(int policy, String srcFile, String dstFile) throws FileNotFoundException {
|
public Maker(int policy, String srcPath, String dstPath) throws IOException {
|
||||||
this.srcFile = new File(srcFile);
|
this.srcFile = new File(srcPath);
|
||||||
if (!this.srcFile.exists()) {
|
if (!this.srcFile.exists()) {
|
||||||
throw new FileNotFoundException("source text file `" +srcFile+ "` not found");
|
throw new FileNotFoundException("source text file `" + srcPath + "` not found");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// check and delete the target xdb file if it exists
|
||||||
|
/// final File dstFile = new File(dstPath);
|
||||||
|
/// if (dstFile.exists() && !dstFile.delete()) {
|
||||||
|
/// log.warnf("failed to delete the dest xdb file `%s`", dstPath);
|
||||||
|
/// }
|
||||||
|
|
||||||
this.bytesCharset = Charset.forName("utf-8");
|
this.bytesCharset = Charset.forName("utf-8");
|
||||||
this.segments = new LinkedList<Segment>();
|
this.segments = new LinkedList<Segment>();
|
||||||
this.dstHandle = new RandomAccessFile(dstFile, "rw");
|
this.dstHandle = new RandomAccessFile(dstPath, "rw");
|
||||||
this.indexPolicy = policy;
|
this.indexPolicy = policy;
|
||||||
this.regionPool = new HashMap<String, DataEntry>();
|
this.regionPool = new HashMap<String, DataEntry>();
|
||||||
this.vectorIndex = new byte[VectorIndexLength]; // all filled with 0
|
this.vectorIndex = new byte[VectorIndexLength]; // all filled with 0
|
||||||
|
|
||||||
|
// truncate the original xdb file
|
||||||
|
this.dstHandle.setLength(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
// init the header of the target xdb binary file
|
// init the header of the target xdb binary file
|
||||||
|
|
@ -122,7 +131,7 @@ public class Maker {
|
||||||
// load all the segments
|
// load all the segments
|
||||||
private void loadSegments() throws Exception {
|
private void loadSegments() throws Exception {
|
||||||
log.infof("try to load the segments ... ");
|
log.infof("try to load the segments ... ");
|
||||||
long tStart = System.currentTimeMillis();
|
final long tStart = System.currentTimeMillis();
|
||||||
Segment last = null;
|
Segment last = null;
|
||||||
String line;
|
String line;
|
||||||
|
|
||||||
|
|
@ -136,8 +145,8 @@ public class Maker {
|
||||||
throw new Exception("invalid ip segment line `"+ps[0]+"`");
|
throw new Exception("invalid ip segment line `"+ps[0]+"`");
|
||||||
}
|
}
|
||||||
|
|
||||||
long sip = Util.checkIP(ps[0]);
|
final long sip = Util.checkIP(ps[0]);
|
||||||
long eip = Util.checkIP(ps[1]);
|
final long eip = Util.checkIP(ps[1]);
|
||||||
if (sip > eip) {
|
if (sip > eip) {
|
||||||
br.close();
|
br.close();
|
||||||
throw new Exception("start ip("+ps[0]+") should not be greater than end ip("+ps[1]+")");
|
throw new Exception("start ip("+ps[0]+") should not be greater than end ip("+ps[1]+")");
|
||||||
|
|
@ -156,7 +165,7 @@ public class Maker {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Segment seg = new Segment(sip, eip, ps[2]);
|
final Segment seg = new Segment(sip, eip, ps[2]);
|
||||||
segments.add(seg);
|
segments.add(seg);
|
||||||
last = seg;
|
last = seg;
|
||||||
}
|
}
|
||||||
|
|
@ -176,10 +185,10 @@ public class Maker {
|
||||||
|
|
||||||
// set the vector index info of the specified ip
|
// set the vector index info of the specified ip
|
||||||
private void setVectorIndex(long ip, long ptr) {
|
private void setVectorIndex(long ip, long ptr) {
|
||||||
int il0 = (int) ((ip >> 24) & 0xFF);
|
final int il0 = (int) ((ip >> 24) & 0xFF);
|
||||||
int il1 = (int) ((ip >> 16) & 0xFF);
|
final int il1 = (int) ((ip >> 16) & 0xFF);
|
||||||
int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
|
final int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
|
||||||
long sPtr = Util.getIntLong(vectorIndex, idx);
|
final long sPtr = Util.getIntLong(vectorIndex, idx);
|
||||||
if (sPtr == 0) {
|
if (sPtr == 0) {
|
||||||
Util.write(vectorIndex, idx, ptr, 4);
|
Util.write(vectorIndex, idx, ptr, 4);
|
||||||
Util.write(vectorIndex, idx + 4, ptr + SegmentIndexSize, 4);
|
Util.write(vectorIndex, idx + 4, ptr + SegmentIndexSize, 4);
|
||||||
|
|
@ -200,14 +209,14 @@ public class Maker {
|
||||||
log.infof("try to write the data block ... ");
|
log.infof("try to write the data block ... ");
|
||||||
for (Segment seg : segments) {
|
for (Segment seg : segments) {
|
||||||
log.infof("try to write region `%s` ... ", seg.region);
|
log.infof("try to write region `%s` ... ", seg.region);
|
||||||
DataEntry e = regionPool.get(seg.region);
|
final DataEntry e = regionPool.get(seg.region);
|
||||||
if (e != null) {
|
if (e != null) {
|
||||||
log.infof(" --[Cached] with ptr=%d", e.ptr);
|
log.infof(" --[Cached] with ptr=%d", e.ptr);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// get the utf-8 bytes of the region info
|
// get the utf-8 bytes of the region info
|
||||||
byte[] regionBuff = seg.region.getBytes(bytesCharset);
|
final byte[] regionBuff = seg.region.getBytes(bytesCharset);
|
||||||
if (regionBuff.length < 1) {
|
if (regionBuff.length < 1) {
|
||||||
throw new Exception("empty region info for segment `"+seg+"`");
|
throw new Exception("empty region info for segment `"+seg+"`");
|
||||||
} else if (regionBuff.length > 0xFFFF) {
|
} else if (regionBuff.length > 0xFFFF) {
|
||||||
|
|
@ -215,7 +224,7 @@ public class Maker {
|
||||||
}
|
}
|
||||||
|
|
||||||
// record the current ptr
|
// record the current ptr
|
||||||
long pos = dstHandle.getFilePointer();
|
final long pos = dstHandle.getFilePointer();
|
||||||
dstHandle.write(regionBuff);
|
dstHandle.write(regionBuff);
|
||||||
|
|
||||||
// record the mapping
|
// record the mapping
|
||||||
|
|
@ -227,7 +236,7 @@ public class Maker {
|
||||||
log.infof("try to write the segment index block ... ");
|
log.infof("try to write the segment index block ... ");
|
||||||
int counter = 0;
|
int counter = 0;
|
||||||
long startIndexPtr = -1, endIndexPtr = -1;
|
long startIndexPtr = -1, endIndexPtr = -1;
|
||||||
byte[] indexBuff = new byte[SegmentIndexSize]; // 4 + 4 + 2 + 4
|
final byte[] indexBuff = new byte[SegmentIndexSize]; // 4 + 4 + 2 + 4
|
||||||
for (Segment seg : segments) {
|
for (Segment seg : segments) {
|
||||||
// we need the region ptr
|
// we need the region ptr
|
||||||
DataEntry e = regionPool.get(seg.region);
|
DataEntry e = regionPool.get(seg.region);
|
||||||
|
|
@ -282,8 +291,8 @@ public class Maker {
|
||||||
}
|
}
|
||||||
|
|
||||||
private static class DataEntry {
|
private static class DataEntry {
|
||||||
long ptr;
|
final long ptr;
|
||||||
int length; // in bytes
|
final int length; // in bytes
|
||||||
|
|
||||||
DataEntry(int length, long ptr) {
|
DataEntry(int length, long ptr) {
|
||||||
this.length = length;
|
this.length = length;
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,7 @@ public class Segment {
|
||||||
|
|
||||||
// parser the Segment from an input string
|
// parser the Segment from an input string
|
||||||
public static Segment parse(String input) throws Exception {
|
public static Segment parse(String input) throws Exception {
|
||||||
String[] ps = input.trim().split("\\|", 3);
|
final String[] ps = input.trim().split("\\|", 3);
|
||||||
if (ps.length != 3) {
|
if (ps.length != 3) {
|
||||||
throw new Exception("invalid ip segment `"+input+"`");
|
throw new Exception("invalid ip segment `"+input+"`");
|
||||||
}
|
}
|
||||||
|
|
@ -39,8 +39,8 @@ public class Segment {
|
||||||
|
|
||||||
// split the current segment for vector index
|
// split the current segment for vector index
|
||||||
public List<Segment> split() {
|
public List<Segment> split() {
|
||||||
long sByte1 = (int) ((startIP >> 24) & 0xFF);
|
final long sByte1 = (int) ((startIP >> 24) & 0xFF);
|
||||||
long eByte1 = (int) ((endIP >> 24) & 0xFF);
|
final long eByte1 = (int) ((endIP >> 24) & 0xFF);
|
||||||
long nSip = startIP;
|
long nSip = startIP;
|
||||||
final List<Segment> tList = new ArrayList<Segment>();
|
final List<Segment> tList = new ArrayList<Segment>();
|
||||||
for (long i = sByte1; i <= eByte1; i++) {
|
for (long i = sByte1; i <= eByte1; i++) {
|
||||||
|
|
@ -59,10 +59,10 @@ public class Segment {
|
||||||
// 2, split the segments with the second byte
|
// 2, split the segments with the second byte
|
||||||
final List<Segment> segList = new ArrayList<Segment>();
|
final List<Segment> segList = new ArrayList<Segment>();
|
||||||
for (Segment seg : tList) {
|
for (Segment seg : tList) {
|
||||||
long base = seg.startIP & 0xFF000000;
|
final long base = seg.startIP & 0xFF000000;
|
||||||
|
final long sb2 = (seg.startIP >> 16) & 0xFF;
|
||||||
|
final long eb2 = (seg.endIP >> 16) & 0xFF;
|
||||||
long tSip = seg.startIP;
|
long tSip = seg.startIP;
|
||||||
long sb2 = (seg.startIP >> 16) & 0xFF;
|
|
||||||
long eb2 = (seg.endIP >> 16) & 0xFF;
|
|
||||||
for (long i = sb2; i <= eb2; i++) {
|
for (long i = sb2; i <= eb2; i++) {
|
||||||
long sip = base | (i << 16) | (tSip & 0xFFFF);
|
long sip = base | (i << 16) | (tSip & 0xFFFF);
|
||||||
long eip = base | (i << 16) | 0xFFFF;
|
long eip = base | (i << 16) | 0xFFFF;
|
||||||
|
|
|
||||||
|
|
@ -51,7 +51,7 @@ public class Util
|
||||||
|
|
||||||
/* check the specified ip address */
|
/* check the specified ip address */
|
||||||
public static long checkIP(String ip) throws Exception {
|
public static long checkIP(String ip) throws Exception {
|
||||||
String[] ps = ip.split("\\.");
|
final String[] ps = ip.split("\\.");
|
||||||
if (ps.length != 4) {
|
if (ps.length != 4) {
|
||||||
throw new Exception("invalid ip address `" + ip + "`");
|
throw new Exception("invalid ip address `" + ip + "`");
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue