xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

maker segment and index policy manager

This commit is contained in:
lion 2022-07-14 16:38:26 +08:00
parent 3488d10d2b
commit d5ceb4cd17
5 changed files with 373 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
maker/java/src/main/java/org/lionsoul/ip2region/xdb/IndexPolicy.java Normal file
View File

@ -0,0 +1,25 @@
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
//
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/07/14
package org.lionsoul.ip2region.xdb;
public class IndexPolicy {
public static final int Vector = 1;
public static final int BTree = 2;
// parser the index policy from string
public static int parse(String policy) throws Exception {
String v = policy.toLowerCase();
if ("vector".equals(v)) {
return Vector;
} else if ("btree".equals(v)) {
return BTree;
} else {
throw new Exception("unknown index policy `"+policy+"`");
}
}
}

116
maker/java/src/main/java/org/lionsoul/ip2region/xdb/Log.java Normal file
View File

@ -0,0 +1,116 @@
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
//
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/07/14
package org.lionsoul.ip2region.xdb;
import java.text.SimpleDateFormat;
import java.util.Date;
// simple log implementation
public class Log {
/* Log level constants define */
public static final int DEBUG = 0;
public static final int INFO = 1;
public static final int WARN = 2;
public static final int ERROR = 3;
// level name
public static final String[] level_string = new String[] {
"DEBUG",
"INFO",
"WARN",
"ERROR"
};
public final Class<?> baseClass;
private static int level;
public Log(Class<?> baseClass) {
this.baseClass = baseClass;
}
public static Log getLogger(Class<?> baseClass) {
return new Log(baseClass);
}
public String format(int level, String format, Object... args) {
// append the datetime
final StringBuilder sb = new StringBuilder();
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
sb.append(String.format("%s %-5s ", sdf.format(new Date()), level_string[level]));
// append the class name
sb.append(baseClass.getName()).append(' ');
sb.append(String.format(format, args));
return sb.toString();
}
public void printf(int level, String format, Object... args) {
if (level < DEBUG || level > ERROR) {
throw new IndexOutOfBoundsException("invalid level index " + level);
}
// level filter
if (level < Log.level) {
return;
}
System.out.println(format(level, format, args));
System.out.flush();
}
public String getDebugf(String format, Object... args) {
return format(DEBUG, format, args);
}
public void debugf(String format, Object... args) {
printf(DEBUG, format, args);
}
public String getInfof(String format, Object... args) {
return format(INFO, format, args);
}
public void infof(String format, Object... args) {
printf(INFO, format, args);
}
public String getWarnf(String format, Object... args) {
return format(WARN, format, args);
}
public void warnf(String format, Object... args) {
printf(WARN, format, args);
}
public String getErrorf(String format, Object... args) {
return format(ERROR, format, args);
}
public void errorf(String format, Object... args) {
printf(ERROR, format, args);
}
public static void setLevel(int level) {
Log.level = level;
}
public static void setLevel(String level) {
String v = level.toLowerCase();
if ("debug".equals(v)) {
Log.level = DEBUG;
} else if ("info".equals(v)) {
Log.level = INFO;
} else if ("warn".equals(v)) {
Log.level = WARN;
} else if ("error".equals(v)) {
Log.level = ERROR;
}
}
}

76
maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java
View File

@ -51,6 +51,13 @@
package org.lionsoul.ip2region.xdb; package org.lionsoul.ip2region.xdb;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.HashMap;
import java.util.Map;
public class Maker { public class Maker {
// constants define // constants define
public static final int VersionNo = 2; public static final int VersionNo = 2;
@ -60,7 +67,74 @@ public class Maker {
public static final int VectorIndexSize = 8; public static final int VectorIndexSize = 8;
public static final int SegmentIndexSize = 14; public static final int SegmentIndexSize = 14;
public Maker() { private static final Log log = Log.getLogger(Maker.class);
// source text file handle
private final File srcFile;
// destination binary file handle
private final RandomAccessFile dstHandle;
// index policy
private final int indexPolicy;
// region pool
private final Map<String, Long> regionPool;
// vector index raw bytes
private byte[] vectorIndex;
public Maker(int policy, String srcFile, String dstFile) throws FileNotFoundException {
this.srcFile = new File(srcFile);
if (!this.srcFile.exists()) {
throw new FileNotFoundException("source text file `" +srcFile+ "` not found");
}
this.dstHandle = new RandomAccessFile(dstFile, "r");
this.indexPolicy = policy;
this.regionPool = new HashMap<String, Long>();
}
// init the header of the target xdb binary file
private void initHeader() throws IOException {
log.infof("try to init the db header ... ");
dstHandle.seek(0);
// make and write the header space
final byte[] header = new byte[HeaderInfoLength];
// encode the data
Util.write(header, 0, VersionNo, 2);
Util.write(header, 2, indexPolicy, 2);
Util.write(header, 4, System.currentTimeMillis() / 1000, 4);
Util.write(header, 8, 0, 4); // start index ptr
Util.write(header, 12, 0, 4); // end index ptr
dstHandle.write(header);
}
// load all the segments
private void loadSegments() {
}
// init the maker
public void init() throws IOException {
// init the db header
initHeader();
// load all the segments
loadSegments();
}
// start to make the binary file
public void make() {
}
// end the make, do the resource clean up
public void end() throws IOException {
this.dstHandle.close();
} }
} }

85
maker/java/src/main/java/org/lionsoul/ip2region/xdb/Segment.java Normal file
View File

@ -0,0 +1,85 @@
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
//
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/07/14
package org.lionsoul.ip2region.xdb;
import java.util.ArrayList;
import java.util.List;
public class Segment {
public final long startIP;
public final long endIP;
public final String region;
// parser the Segment from an input string
public static Segment parse(String input) throws Exception {
String[] ps = input.split("\\|", 3);
if (ps.length != 3) {
throw new Exception("invalid ip segment `"+input+"`");
}
long sip = Util.checkIP(ps[0]);
long eip = Util.checkIP(ps[1]);
if (sip > eip) {
throw new Exception("start ip `"+ps[0]+"` should not be greater than end ip `"+ps[1]+"`");
}
return new Segment(sip, eip, ps[2]);
}
public Segment(long startIP, long endIP, String region) {
this.startIP = startIP;
this.endIP = endIP;
this.region = region;
}
// split the current segment for vector index
public List<Segment> split() {
long sByte1 = (int) ((startIP >> 24) & 0xFF);
long eByte1 = (int) ((endIP >> 24) & 0xFF);
long nSip = startIP;
final List<Segment> tList = new ArrayList<Segment>();
for (long i = sByte1; i <= eByte1; i++) {
long sip = (i << 24) | (nSip & 0xFFFFFF);
long eip = (i << 24) | 0xFFFFFF;
if (eip < endIP) {
nSip = (i + 1) << 24;
} else {
eip = endIP;
}
// append the new segment
tList.add(new Segment(sip, eip, null));
}
// 2, split the segments with the second byte
final List<Segment> segList = new ArrayList<Segment>();
for (Segment seg : tList) {
long base = seg.startIP & 0xFF000000;
long tSip = seg.startIP;
long sb2 = (seg.startIP >> 16) & 0xFF;
long eb2 = (seg.endIP >> 16) & 0xFF;
for (long i = sb2; i <= eb2; i++) {
long sip = base | (i << 16) | (tSip & 0xFFFF);
long eip = base | (i << 16) | 0xFFFF;
if (eip < seg.endIP) {
tSip = 0;
} else {
eip = seg.endIP;
}
segList.add(new Segment(sip, eip, seg.region));
}
}
return segList;
}
@Override public String toString() {
return Util.long2ip(startIP) + "|" + Util.long2ip(endIP) + "|" + region;
}
}

72
maker/java/src/main/java/org/lionsoul/ip2region/xdb/Util.java Normal file
View File

@ -0,0 +1,72 @@
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
//
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/07/14
package org.lionsoul.ip2region.xdb;
public class Util
{
// write specified bytes into a byte array start from offset
public static void write( byte[] b, int offset, long v, int bytes) {
for ( int i = 0; i < bytes; i++ ) {
b[offset++] = (byte)((v >>> (8 * i)) & 0xFF);
}
}
// write a int to a byte array
public static void writeIntLong(byte[] b, int offset, long v) {
b[offset++] = (byte)((v ) & 0xFF);
b[offset++] = (byte)((v >> 8) & 0xFF);
b[offset++] = (byte)((v >> 16) & 0xFF);
b[offset ] = (byte)((v >> 24) & 0xFF);
}
// get an int from a byte array start from the specified offset
public static long getIntLong(byte[] b, int offset) {
return (
((b[offset++] & 0x000000FFL)) |
((b[offset++] << 8) & 0x0000FF00L) |
((b[offset++] << 16) & 0x00FF0000L) |
((b[offset ] << 24) & 0xFF000000L)
);
}
public static int getInt2(byte[] b, int offset) {
return (
(b[offset++] & 0x000000FF) |
(b[offset ] & 0x0000FF00)
);
}
/* long int to ip string */
public static String long2ip( long ip ) {
return String.valueOf((ip >> 24) & 0xFF) + '.' +
((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF);
}
public static final byte[] shiftIndex = {24, 16, 8, 0};
/* check the specified ip address */
public static long checkIP(String ip) throws Exception {
String[] ps = ip.split("\\.");
if (ps.length != 4) {
throw new Exception("invalid ip address `" + ip + "`");
}
long ipDst = 0;
for (int i = 0; i < ps.length; i++) {
int val = Integer.parseInt(ps[i]);
if (val > 255) {
throw new Exception("ip part `"+ps[i]+"` should be less then 256");
}
ipDst |= ((long) val << shiftIndex[i]);
}
return ipDst & 0xFFFFFFFFL;
}
}