diff --git a/binding/java/ReadMe.md b/binding/java/ReadMe.md index ca5ac0b..f4ba058 100644 --- a/binding/java/ReadMe.md +++ b/binding/java/ReadMe.md @@ -11,140 +11,57 @@ ``` -### 完全基于文件的查询 +### Example ```java +package org.lionsoul.ip2region.example; + import org.lionsoul.ip2region.xdb.Searcher; -import java.io.*; + import java.util.concurrent.TimeUnit; -public class SearcherTest { - public static void main(String[] args) { - // 1、创建 searcher 对象 +/** + * @see org.lionsoul.ip2region.xdb.Searcher + */ +public class SearcherExample { + + public static void main(String[] args) throws Exception { String dbPath = "ip2region.xdb file path"; - Searcher searcher = null; - try { - searcher = Searcher.newWithFileOnly(dbPath); - } catch (IOException e) { - System.out.printf("failed to create searcher with `%s`: %s\n", dbPath, e); - return; + + // 基于文件查询，单次 search 需要文件 IO，非线程安全，性能最差 + try (Searcher fileSearcher = Searcher.newWithFileOnly(dbPath)) { + searchExample(fileSearcher); } - // 2、查询 + // 基于 VectorIndex 查询，单次 search 需要内存查询 + 文件 IO，非线程安全，性能次之 + try (Searcher indexSearcher = Searcher.newWithVectorIndex(dbPath)) { + searchExample(indexSearcher); + } + + // 推荐：基于 Buffer 查询，单次 search 全使用内存查询，线程安全，支持序列化，性能最佳 + // 可重写 loadFile 方法扩展从其他文件系统(hdfs/s3/oss...)装载 buffer 数据 + try (Searcher bufferSearcher = Searcher.newWithBuffer(dbPath)) { + searchExample(bufferSearcher); + } + + // 备注：并发查询时，对于非线程安全实现推荐为每个线程创建一个独立的 searcher 对象单独使用。 + } + + private static void searchExample(Searcher searcher) { + String ip = "1.2.3.4"; try { - String ip = "1.2.3.4"; long sTime = System.nanoTime(); - String region = searcher.search(ip); - long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); - System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); + Searcher.Region region = searcher.searchRegion(ip); + long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime); + System.out.printf("{region: %s, searcher: %s, ioCount: %d, took: %d μs}\n", + region, searcher.getClass().getSimpleName(), region.getIoCount(), cost); + System.out.println(region.toRegionMsg()); } catch (Exception e) { System.out.printf("failed to search(%s): %s\n", ip, e); } - - // 3、关闭资源 - searcher.close(); - - // 备注：并发使用，每个线程需要创建一个独立的 searcher 对象单独使用。 } } -``` -### 缓存 `VectorIndex` 索引 - -我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据，然后全局缓存，每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作，从而加速查询，减少 IO 压力。 -```java -import org.lionsoul.ip2region.xdb.Searcher; -import java.io.*; -import java.util.concurrent.TimeUnit; - -public class SearcherTest { - public static void main(String[] args) { - String dbPath = "ip2region.xdb file path"; - - // 1、从 dbPath 中预先加载 VectorIndex 缓存，并且把这个得到的数据作为全局变量，后续反复使用。 - byte[] vIndex; - try { - vIndex = Searcher.loadVectorIndexFromFile(dbPath); - } catch (Exception e) { - System.out.printf("failed to load vector index from `%s`: %s\n", dbPath, e); - return; - } - - // 2、使用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。 - Searcher searcher; - try { - searcher = Searcher.newWithVectorIndex(dbPath, vIndex); - } catch (Exception e) { - System.out.printf("failed to create vectorIndex cached searcher with `%s`: %s\n", dbPath, e); - return; - } - - // 3、查询 - try { - String ip = "1.2.3.4"; - long sTime = System.nanoTime(); - String region = searcher.search(ip); - long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); - System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); - } catch (Exception e) { - System.out.printf("failed to search(%s): %s\n", ip, e); - } - - // 4、关闭资源 - searcher.close(); - - // 备注：每个线程需要单独创建一个独立的 Searcher 对象，但是都共享全局的制度 vIndex 缓存。 - } -} -``` - -### 缓存整个 `xdb` 数据 - -我们也可以预先加载整个 ip2region.xdb 的数据到内存，然后基于这个数据创建查询对象来实现完全基于文件的查询，类似之前的 memory search。 -```java -import org.lionsoul.ip2region.xdb.Searcher; -import java.io.*; -import java.util.concurrent.TimeUnit; - -public class SearcherTest { - public static void main(String[] args) { - String dbPath = "ip2region.xdb file path"; - - // 1、从 dbPath 加载整个 xdb 到内存。 - byte[] cBuff; - try { - cBuff = Searcher.loadContentFromFile(dbPath); - } catch (Exception e) { - System.out.printf("failed to load content from `%s`: %s\n", dbPath, e); - return; - } - - // 2、使用上述的 cBuff 创建一个完全基于内存的查询对象。 - Searcher searcher; - try { - searcher = Searcher.newWithBuffer(cBuff); - } catch (Exception e) { - System.out.printf("failed to create content cached searcher: %s\n", e); - return; - } - - // 3、查询 - try { - String ip = "1.2.3.4"; - long sTime = System.nanoTime(); - String region = searcher.search(ip); - long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); - System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); - } catch (Exception e) { - System.out.printf("failed to search(%s): %s\n", ip, e); - } - - // 4、关闭资源 - 该 searcher 对象可以安全用于并发，等整个服务关闭的时候再关闭 searcher - // searcher.close(); - - // 备注：并发使用，用整个 xdb 数据缓存创建的查询对象可以安全的用于并发，也就是你可以把这个 searcher 对象做成全局对象去跨线程访问。 - } -} ``` @@ -174,7 +91,7 @@ options: 例如：使用默认的 data/ip2region.xdb 文件进行查询测试： ```bash -➜ java git:(v2.0_xdb) ✗ java -jar target/ip2region-2.6.0.jar search --db=../../data/ip2region.xdb +$ java -jar target/ip2region-2.6.5.jar search --db=../../data/ip2region.xdb ip2region xdb searcher test program, cachePolicy: vectorIndex type 'quit' to exit ip2region>> 1.2.3.4 @@ -182,26 +99,31 @@ ip2region>> 1.2.3.4 ip2region>> ``` -输入 ip 即可进行查询测试，也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的查询效果。 +输入 ip 即可进行查询测试，也可以分别设置 `--cache-policy` 来测试三种不同缓存实现的查询效果。 # bench 测试 可以通过 `java -jar ip2region-{version}.jar bench` 命令来进行 bench 测试，一方面确保 `xdb` 文件没有错误，一方面可以评估查询性能： ```bash -➜ java git:(v2.0_xdb) ✗ java -jar target/ip2region-2.6.0.jar bench +java -jar target/ip2region-2.6.5.jar bench + java -jar ip2region-{version}.jar bench [command options] options: --db string ip2region binary xdb file path --src string source ip text file path - --cache-policy string cache policy: file/vectorIndex/content + --cache-policy string cache policy: file/vectorIndex/buffer ``` 例如：通过默认的 data/ip2region.xdb 和 data/ip.merge.txt 文件进行 bench 测试： ```bash -➜ java git:(v2.0_xdb) ✗ java -jar target/ip2region-2.6.0.jar bench --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt -Bench finished, {cachePolicy: vectorIndex, total: 3417955, took: 8s, cost: 2 μs/op} +$ java -jar target/ip2region-2.6.5.jar bench --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt --cache-policy=buffer + +-- 笔者笔记本[mac 2.6 GHz 六核Intel Core i7]测试结果案例 +Bench finished, {cachePolicy: buffer, total: 3417955, took: 2s, ioCount: 0, cost: 0 μs/op} +Bench finished, {cachePolicy: vectorIndex, total: 3417955, took: 28s, ioCount: 43465554, cost: 8 μs/op} +Bench finished, {cachePolicy: file, total: 3417955, took: 34s, ioCount: 43465554, cost: 9 μs/op} ``` -可以通过分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效果。 +可以通过分别设置 `--cache-policy` 来测试三种不同缓存实现的效果。 @Note: 注意 bench 使用的 src 文件要是生成对应 xdb 文件相同的源文件。 diff --git a/binding/java/pom.xml b/binding/java/pom.xml index babaaee..80860b1 100644 --- a/binding/java/pom.xml +++ b/binding/java/pom.xml @@ -44,14 +44,38 @@ + + org.projectlombok + lombok + 1.18.24 + provided + junit junit - 4.13.1 + 4.13.2 test + + net.jcip + jcip-annotations + 1.0 + provided + + + + aliyun + https://maven.aliyun.com/repository/public + + + + repo1 + https://repo1.maven.org/maven2/ + + + @@ -103,7 +127,7 @@ - org.lionsoul.ip2region.SearchTest + org.lionsoul.ip2region.xdb.SearchTest diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/example/SearcherExample.java b/binding/java/src/main/java/org/lionsoul/ip2region/example/SearcherExample.java new file mode 100644 index 0000000..a4e4d3d --- /dev/null +++ b/binding/java/src/main/java/org/lionsoul/ip2region/example/SearcherExample.java @@ -0,0 +1,47 @@ +package org.lionsoul.ip2region.example; + +import org.lionsoul.ip2region.xdb.Searcher; + +import java.util.concurrent.TimeUnit; + +/** + * @see org.lionsoul.ip2region.xdb.Searcher + */ +public class SearcherExample { + + public static void main(String[] args) throws Exception { + String dbPath = "ip2region.xdb file path"; + + // 基于文件查询，单次 search 需要文件 IO，非线程安全，性能最差 + try (Searcher fileSearcher = Searcher.newWithFileOnly(dbPath)) { + searchExample(fileSearcher); + } + + // 基于 VectorIndex 查询，单次 search 需要内存查询 + 文件 IO，非线程安全，性能次之 + try (Searcher indexSearcher = Searcher.newWithVectorIndex(dbPath)) { + searchExample(indexSearcher); + } + + // 推荐：基于 Buffer 查询，单次 search 全使用内存查询，线程安全，支持序列化，性能最佳 + // 可重写 loadFile 方法扩展从其他文件系统(hdfs/s3/oss...)装载 buffer 数据 + try (Searcher bufferSearcher = Searcher.newWithBuffer(dbPath)) { + searchExample(bufferSearcher); + } + + // 备注：并发查询时，对于非线程安全实现推荐为每个线程创建一个独立的 searcher 对象单独使用。 + } + + private static void searchExample(Searcher searcher) { + String ip = "1.2.3.4"; + try { + long sTime = System.nanoTime(); + Searcher.Region region = searcher.searchRegion(ip); + long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime); + System.out.printf("{region: %s, searcher: %s, ioCount: %d, took: %d μs}\n", + region, searcher.getClass().getSimpleName(), region.getIoCount(), cost); + System.out.println(region.toRegionMsg()); + } catch (Exception e) { + System.out.printf("failed to search(%s): %s\n", ip, e); + } + } +} diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/AbstractSearcher.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/AbstractSearcher.java new file mode 100644 index 0000000..b949dec --- /dev/null +++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/AbstractSearcher.java @@ -0,0 +1,106 @@ +package org.lionsoul.ip2region.xdb; + +import java.nio.charset.StandardCharsets; + +import static org.lionsoul.ip2region.xdb.InternalUtil.*; + +/** + * @author Li.Wei by 2022/8/6 + */ +public abstract class AbstractSearcher implements Searcher { + // constant defined copied from the xdb maker + protected static final int HEADER_INFO_LENGTH = 256; + protected static final int VECTOR_INDEX_ROWS = 256; + protected static final int VECTOR_INDEX_COLS = 256; + protected static final int VECTOR_INDEX_SIZE = 8; + protected static final int SEGMENT_INDEX_SIZE = 14; + + @Deprecated + public int getIOCount() { + return 0; + } + + public Region searchRegion(String ip) { + return search0(ip2long(ip)); + } + + public String search(String ip) { + return searchRegion(ip).getRegion(); + } + + private Region search0(long ip) { + int ioCount = 0; + + // locate the segment index block based on the vector index + int il0 = (int) ((ip >> 24) & 0xFF); + int il1 = (int) ((ip >> 16) & 0xFF); + int idx = il0 * VECTOR_INDEX_COLS * VECTOR_INDEX_SIZE + il1 * VECTOR_INDEX_SIZE; + // System.out.printf("il0: %d, il1: %d, idx: %d\n", il0, il1, idx); + PointIndex pointIndex = this.pointIndex(idx); + int sPtr = pointIndex.sPtr, ePtr = pointIndex.ePtr; + // System.out.printf("sPtr: %d, ePtr: %d\n", sPtr, ePtr); + + // binary search the segment index block to get the region info + final byte[] buff = new byte[SEGMENT_INDEX_SIZE]; + int dataLen = -1, dataPtr = -1; + int l = 0, h = (ePtr - sPtr) / SEGMENT_INDEX_SIZE; + while (l <= h) { + int m = (l + h) >> 1; + int p = sPtr + m * SEGMENT_INDEX_SIZE; + + // read the segment index + ioCount += read(p, buff); + + long sip = getIntLong(buff, 0); + if (ip < sip) { + h = m - 1; + } else { + long eip = getIntLong(buff, 4); + if (ip > eip) { + l = m + 1; + } else { + dataLen = getInt2(buff, 8); + dataPtr = getInt(buff, 10); + break; + } + } + } + + // empty match interception + // System.out.printf("dataLen: %d, dataPtr: %d\n", dataLen, dataPtr); + if (dataPtr < 0) { + return Region.builder().ioCount(ioCount).build(); + } + + // load and return the region data + final byte[] regionBuff = new byte[dataLen]; + ioCount += read(dataPtr, regionBuff); + return Region.builder().region(new String(regionBuff, StandardCharsets.UTF_8)).ioCount(ioCount).build(); + } + + /** + * find index start,end point + * + * @param idx idx + * @return PointIndex + */ + protected abstract PointIndex pointIndex(int idx); + + /** + * read 2 buffer + * + * @param offset offset + * @param buffer buffer + * @return io count + */ + protected abstract int read(int offset, byte[] buffer); + + protected static class PointIndex { + int sPtr, ePtr; + + public PointIndex(int sPtr, int ePtr) { + this.sPtr = sPtr; + this.ePtr = ePtr; + } + } +} diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/BufferSearcher.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/BufferSearcher.java new file mode 100644 index 0000000..6aaed80 --- /dev/null +++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/BufferSearcher.java @@ -0,0 +1,73 @@ +package org.lionsoul.ip2region.xdb; + +import net.jcip.annotations.ThreadSafe; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.io.Serializable; + +import static org.lionsoul.ip2region.xdb.InternalUtil.getInt; + +/** + * xdb content buffer, used for in-memory search. + */ +@ThreadSafe +public class BufferSearcher extends AbstractSearcher implements Serializable { + + private final byte[] contentBuff; + + public BufferSearcher(byte[] contentBuff) { + this.contentBuff = contentBuff; + } + + public BufferSearcher(String dbFile) { + try { + this.contentBuff = loadFile(dbFile); + } catch (IOException e) { + throw new SearcherException("load dbFile error: " + e.getMessage() + ", file=" + dbFile, e); + } + } + + /** + * default load local file. override load hdfs/s3/oss... + * + * @param dbFile dbFile + * @return byte array + * @throws IOException ex + */ + protected byte[] loadFile(String dbFile) throws IOException { + return loadLocalFile(dbFile); + } + + @Override + protected int read(int offset, byte[] buffer) { + System.arraycopy(contentBuff, offset, buffer, 0, buffer.length); + return 0; + } + + @Override + protected PointIndex pointIndex(int idx) { + return new PointIndex( + getInt(contentBuff, HEADER_INFO_LENGTH + idx), + getInt(contentBuff, HEADER_INFO_LENGTH + idx + 4) + ); + } + + @Override + public void close() { + // nothing + } + + private static byte[] loadLocalFile(String dbPath) throws IOException { + final byte[] buff; + try (RandomAccessFile handle = new RandomAccessFile(dbPath, "r")) { + handle.seek(0); + buff = new byte[(int) handle.length()]; + int rLen = handle.read(buff); + if (rLen != buff.length) { + throw new IOException("incomplete read: read bytes should be " + buff.length); + } + } + return buff; + } +} diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/FileSearcher.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/FileSearcher.java new file mode 100644 index 0000000..72da748 --- /dev/null +++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/FileSearcher.java @@ -0,0 +1,52 @@ +package org.lionsoul.ip2region.xdb; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; + +import static org.lionsoul.ip2region.xdb.InternalUtil.getInt; + +/** + * random access file handle for file based search. + */ +public class FileSearcher extends AbstractSearcher { + private final RandomAccessFile handle; + + public FileSearcher(String dbFile) { + try { + this.handle = new RandomAccessFile(dbFile, "r"); + } catch (FileNotFoundException e) { + throw new SearcherException("load dbFile error: " + e.getMessage() + ", file=" + dbFile, e); + } + } + + @Override + protected int read(int offset, byte[] buffer) { + int rLen; + try { + handle.seek(offset); + rLen = handle.read(buffer); + } catch (SearcherException | IOException e) { + throw new SearcherException("read error", e); + } + if (rLen != buffer.length) { + throw new SearcherException("incomplete read: read bytes should be " + buffer.length); + } + return 2; + } + + @Override + protected PointIndex pointIndex(int idx) { + final byte[] buff = new byte[VECTOR_INDEX_SIZE]; + read(HEADER_INFO_LENGTH + idx, buff); + return new PointIndex( + getInt(buff, 0), + getInt(buff, 4) + ); + } + + @Override + public void close() throws IOException { + this.handle.close(); + } +} diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java index 8ff0680..e651770 100644 --- a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java +++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java @@ -7,9 +7,8 @@ package org.lionsoul.ip2region.xdb; -import java.awt.image.SampleModel; - -public class Header { +// Internal class +class Header { public final int version; public final int indexPolicy; public final int createdAt; @@ -19,21 +18,22 @@ public class Header { public Header(byte[] buff) { assert buff.length >= 16; - version = Searcher.getInt2(buff, 0); - indexPolicy = Searcher.getInt2(buff, 2); - createdAt = Searcher.getInt(buff, 4); - startIndexPtr = Searcher.getInt(buff, 8); - endIndexPtr = Searcher.getInt(buff, 12); + version = InternalUtil.getInt2(buff, 0); + indexPolicy = InternalUtil.getInt2(buff, 2); + createdAt = InternalUtil.getInt(buff, 4); + startIndexPtr = InternalUtil.getInt(buff, 8); + endIndexPtr = InternalUtil.getInt(buff, 12); buffer = buff; } - @Override public String toString() { + @Override + public String toString() { return "{" + - "Version: " + version + ',' + - "IndexPolicy: " + indexPolicy + ',' + - "CreatedAt: " + createdAt + ',' + - "StartIndexPtr: " + startIndexPtr + ',' + - "EndIndexPtr: " + endIndexPtr + - '}'; + "Version: " + version + ',' + + "IndexPolicy: " + indexPolicy + ',' + + "CreatedAt: " + createdAt + ',' + + "StartIndexPtr: " + startIndexPtr + ',' + + "EndIndexPtr: " + endIndexPtr + + '}'; } } diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/InternalUtil.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/InternalUtil.java new file mode 100644 index 0000000..17e2ecc --- /dev/null +++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/InternalUtil.java @@ -0,0 +1,58 @@ +package org.lionsoul.ip2region.xdb; + +/** + * Internal Util + */ +public class InternalUtil { + private static final byte[] SHIFT_INDEX = {24, 16, 8, 0}; + + /* get an int from a byte array start from the specified offset */ + protected static long getIntLong(byte[] b, int offset) { + return ( + ((b[offset++] & 0x000000FFL)) | + ((b[offset++] << 8) & 0x0000FF00L) | + ((b[offset++] << 16) & 0x00FF0000L) | + ((b[offset] << 24) & 0xFF000000L) + ); + } + + protected static int getInt(byte[] b, int offset) { + return ( + ((b[offset++] & 0x000000FF)) | + ((b[offset++] << 8) & 0x0000FF00) | + ((b[offset++] << 16) & 0x00FF0000) | + ((b[offset] << 24) & 0xFF000000) + ); + } + + protected static int getInt2(byte[] b, int offset) { + return ( + (b[offset++] & 0x000000FF) | + (b[offset] & 0x0000FF00) + ); + } + + /* long int to ip string */ + public static String long2ip(long ip) { + return String.valueOf((ip >> 24) & 0xFF) + '.' + + ((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF); + } + + /* check the specified ip address */ + public static long ip2long(String ip) { + String[] ps = ip.split("\\."); + if (ps.length != 4) { + throw new SearcherException("invalid ip address `" + ip + "`"); + } + + long ipDst = 0; + for (int i = 0; i < ps.length; i++) { + int val = Integer.parseInt(ps[i]); + if (val > 255) { + throw new SearcherException("ip part `" + ps[i] + "` should be less then 256"); + } + ipDst |= ((long) val << SHIFT_INDEX[i]); + } + return ipDst & 0xFFFFFFFFL; + } +} diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/SearchTest.java similarity index 62% rename from binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java rename to binding/java/src/main/java/org/lionsoul/ip2region/xdb/SearchTest.java index 5c70266..f501011 100644 --- a/binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java +++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/SearchTest.java @@ -4,16 +4,23 @@ // @Author Lion // @Date 2022/06/23 -package org.lionsoul.ip2region; +package org.lionsoul.ip2region.xdb; -import org.lionsoul.ip2region.xdb.Searcher; - -import java.io.*; +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.concurrent.TimeUnit; +// for test public class SearchTest { - public static void printHelp(String[] args) { + protected static final String FILE = "file"; + protected static final String VECTOR_INDEX = "vectorIndex"; + protected static final String BUFFER = "buffer"; + + public static void printHelp() { System.out.print("ip2region xdb searcher\n"); System.out.print("java -jar ip2region-{version}.jar [command] [command options]\n"); System.out.print("Command: \n"); @@ -21,22 +28,21 @@ public class SearchTest { System.out.print(" bench search bench test\n"); } - public static Searcher createSearcher(String dbPath, String cachePolicy) throws IOException { - if ("file".equals(cachePolicy)) { - return Searcher.newWithFileOnly(dbPath); - } else if ("vectorIndex".equals(cachePolicy)) { - byte[] vIndex = Searcher.loadVectorIndexFromFile(dbPath); - return Searcher.newWithVectorIndex(dbPath, vIndex); - } else if ("content".equals(cachePolicy)) { - byte[] cBuff = Searcher.loadContentFromFile(dbPath); - return Searcher.newWithBuffer(cBuff); - } else { - throw new IOException("invalid cache policy `" + cachePolicy + "`, options: file/vectorIndex/content"); + public static AbstractSearcher createSearcher(String dbPath, String cachePolicy) throws IOException { + switch (cachePolicy) { + case FILE: + return (AbstractSearcher) Searcher.newWithFileOnly(dbPath); + case VECTOR_INDEX: + return (AbstractSearcher) Searcher.newWithVectorIndex(dbPath); + case BUFFER: + return (AbstractSearcher) Searcher.newWithBuffer(dbPath); + default: + throw new IOException("invalid cache policy `" + cachePolicy + "`, see help"); } } - public static void searchTest(String[] args) throws IOException { - String dbPath = "", cachePolicy = "vectorIndex"; + public static void searchTest(String[] args) throws Exception { + String dbPath = "", cachePolicy = VECTOR_INDEX; for (final String r : args) { if (r.length() < 5) { continue; @@ -74,16 +80,16 @@ public class SearchTest { } Searcher searcher = createSearcher(dbPath, cachePolicy); - final BufferedReader reader = new BufferedReader(new InputStreamReader(System.in)); + final BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8)); System.out.printf("ip2region xdb searcher test program, cachePolicy: %s\ntype 'quit' to exit\n", cachePolicy); - while ( true ) { + while (true) { System.out.print("ip2region>> "); String line = reader.readLine().trim(); - if ( line.length() < 2 ) { + if (line.length() < 2) { continue; } - if ( line.equalsIgnoreCase("quit") ) { + if (line.equalsIgnoreCase("quit")) { break; } @@ -91,9 +97,9 @@ public class SearchTest { double sTime = System.nanoTime(); String region = searcher.search(line); long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); - System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); + System.out.printf("{region: %s, took: %d μs}\n", region, cost); } catch (Exception e) { - System.out.printf("{err: %s, ioCount: %d}\n", e, searcher.getIOCount()); + System.out.printf("{err: %s}\n", e); } } @@ -102,8 +108,8 @@ public class SearchTest { System.out.println("searcher test program exited, thanks for trying"); } - public static void benchTest(String[] args) throws IOException { - String dbPath = "", srcPath = "", cachePolicy = "vectorIndex"; + public static void benchTest(String[] args) throws Exception { + String dbPath = "", srcPath = "", cachePolicy = VECTOR_INDEX; for (final String r : args) { if (r.length() < 5) { continue; @@ -121,15 +127,19 @@ public class SearchTest { String key = r.substring(2, sIdx); String val = r.substring(sIdx + 1); - if ("db".equals(key)) { - dbPath = val; - } else if ("src".equals(key)) { - srcPath = val; - } else if ("cache-policy".equals(key)) { - cachePolicy = val; - } else { - System.out.printf("undefined option `%s`\n", r); - return; + switch (key) { + case "db": + dbPath = val; + break; + case "src": + srcPath = val; + break; + case "cache-policy": + cachePolicy = val; + break; + default: + System.out.printf("undefined option `%s`\n", r); + return; } } @@ -138,12 +148,13 @@ public class SearchTest { System.out.print("options:\n"); System.out.print(" --db string ip2region binary xdb file path\n"); System.out.print(" --src string source ip text file path\n"); - System.out.print(" --cache-policy string cache policy: file/vectorIndex/content\n"); + System.out.print(" --cache-policy string cache policy: " + FILE + "/" + VECTOR_INDEX + "/" + BUFFER + "\n"); + return; } - Searcher searcher = createSearcher(dbPath, cachePolicy); - long count = 0, costs = 0, tStart = System.nanoTime(); + AbstractSearcher searcher = createSearcher(dbPath, cachePolicy); + long count = 0, costs = 0, ioCount = 0, tStart = System.nanoTime(); String line; final BufferedReader reader = new BufferedReader(new FileReader(srcPath)); while ((line = reader.readLine()) != null) { @@ -156,7 +167,7 @@ public class SearchTest { long sip; try { - sip = Searcher.checkIP(ps[0]); + sip = InternalUtil.ip2long(ps[0]); } catch (Exception e) { System.out.printf("check start ip `%s`: %s\n", ps[0], e); return; @@ -164,7 +175,7 @@ public class SearchTest { long eip; try { - eip = Searcher.checkIP(ps[1]); + eip = InternalUtil.ip2long(ps[1]); } catch (Exception e) { System.out.printf("check end ip `%s`: %s\n", ps[1], e); return; @@ -178,12 +189,12 @@ public class SearchTest { long mip = (sip + eip) >> 1; for (final long ip : new long[]{sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) { long sTime = System.nanoTime(); - String region = searcher.search(ip); + Searcher.Region region = searcher.searchRegion(InternalUtil.long2ip(ip)); costs += System.nanoTime() - sTime; - + ioCount += region.getIoCount(); // check the region info - if (!ps[2].equals(region)) { - System.out.printf("failed search(%s) with (%s != %s)\n", Searcher.long2ip(ip), region, ps[2]); + if (!ps[2].equals(region.getRegion())) { + System.out.printf("failed search(%s) with (%s != %s)\n", InternalUtil.long2ip(ip), region, ps[2]); return; } @@ -194,31 +205,36 @@ public class SearchTest { reader.close(); searcher.close(); long took = System.nanoTime() - tStart; - System.out.printf("Bench finished, {cachePolicy: %s, total: %d, took: %ds, cost: %d μs/op}\n", + System.out.printf("Bench finished, {cachePolicy: %s, total: %d, took: %ds, ioCount: %d, cost: %d μs/op}\n", cachePolicy, count, TimeUnit.NANOSECONDS.toSeconds(took), - count == 0 ? 0 : TimeUnit.NANOSECONDS.toMicros(costs/count)); + ioCount, + count == 0 ? 0 : TimeUnit.NANOSECONDS.toMicros(costs / count)); } public static void main(String[] args) { if (args.length < 1) { - printHelp(args); + printHelp(); return; } - if ("search".equals(args[0])) { - try { - searchTest(args); - } catch (IOException e) { - System.out.printf("failed running search test: %s\n", e); - } - } else if ("bench".equals(args[0])) { - try { - benchTest(args); - } catch (IOException e) { - System.out.printf("failed running bench test: %s\n", e); - } - } else { - printHelp(args); + switch (args[0]) { + case "search": + try { + searchTest(args); + } catch (Exception e) { + System.out.printf("failed running search test: %s\n", e); + } + break; + case "bench": + try { + benchTest(args); + } catch (Exception e) { + System.out.printf("failed running bench test: %s\n", e); + } + break; + default: + printHelp(); + break; } } diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java index fcdec4c..8d7d6f8 100644 --- a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java +++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java @@ -9,264 +9,76 @@ package org.lionsoul.ip2region.xdb; // @Date 2022/06/23 -import java.io.IOException; -import java.io.RandomAccessFile; +import lombok.Builder; +import lombok.Data; -public class Searcher { - // constant defined copied from the xdb maker - public static final int HeaderInfoLength = 256; - public static final int VectorIndexRows = 256; - public static final int VectorIndexCols = 256; - public static final int VectorIndexSize = 8; - public static final int SegmentIndexSize = 14; +/** + * @see FileSearcher + * @see VectorIndexSearcher + * @see BufferSearcher + */ +public interface Searcher extends AutoCloseable { - // random access file handle for file based search - private final RandomAccessFile handle; - - private int ioCount = 0; - - // vector index. - // use the byte[] instead of VectorIndex entry array to keep - // the minimal memory allocation. - private final byte[] vectorIndex; - - // xdb content buffer, used for in-memory search - private final byte[] contentBuff; - - // --- static method to create searchers - - public static Searcher newWithFileOnly(String dbPath) throws IOException { - return new Searcher(dbPath, null, null); + // FileSearcher + static Searcher newWithFileOnly(String dbPath) { + return new FileSearcher(dbPath); } - public static Searcher newWithVectorIndex(String dbPath, byte[] vectorIndex) throws IOException { - return new Searcher(dbPath, vectorIndex, null); + // IndexSearcher + static Searcher newWithVectorIndex(String dbPath) { + return new VectorIndexSearcher(dbPath); } - public static Searcher newWithBuffer(byte[] cBuff) throws IOException { - return new Searcher(null, null, cBuff); + // IndexSearcher + static Searcher newWithVectorIndex(String dbPath, byte[] vectorIndex) { + return new VectorIndexSearcher(dbPath, vectorIndex); } - // --- End of creator - - public Searcher(String dbFile, byte[] vectorIndex, byte[] cBuff) throws IOException { - if (cBuff != null) { - this.handle = null; - this.vectorIndex = null; - this.contentBuff = cBuff; - } else { - this.handle = new RandomAccessFile(dbFile, "r"); - this.vectorIndex = vectorIndex; - this.contentBuff = null; - } + // BufferSearcher + static Searcher newWithBuffer(String dbPath) { + return new BufferSearcher(dbPath); } - public void close() throws IOException { - if (this.handle != null) { - this.handle.close(); - } + // BufferSearcher + static Searcher newWithBuffer(byte[] cBuff) { + return new BufferSearcher(cBuff); } - public int getIOCount() { - return ioCount; - } + String search(String ip); - public String search(String ipStr) throws Exception { - long ip = checkIP(ipStr); - return search(ip); - } + Region searchRegion(String ip); - public String search(long ip) throws IOException { - // reset the global counter - this.ioCount = 0; + @Data + @Builder + class Region { + private int ioCount; + private String region; - // locate the segment index block based on the vector index - int sPtr = 0, ePtr = 0; - int il0 = (int) ((ip >> 24) & 0xFF); - int il1 = (int) ((ip >> 16) & 0xFF); - int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize; - // System.out.printf("il0: %d, il1: %d, idx: %d\n", il0, il1, idx); - if (vectorIndex != null) { - sPtr = getInt(vectorIndex, idx); - ePtr = getInt(vectorIndex, idx + 4); - } else if (contentBuff != null) { - sPtr = getInt(contentBuff, HeaderInfoLength + idx); - ePtr = getInt(contentBuff, HeaderInfoLength + idx + 4); - } else { - final byte[] buff = new byte[VectorIndexSize]; - read(HeaderInfoLength + idx, buff); - sPtr = getInt(buff, 0); - ePtr = getInt(buff, 4); - } - - // System.out.printf("sPtr: %d, ePtr: %d\n", sPtr, ePtr); - - // binary search the segment index block to get the region info - final byte[] buff = new byte[SegmentIndexSize]; - int dataLen = -1, dataPtr = -1; - int l = 0, h = (ePtr - sPtr) / SegmentIndexSize; - while (l <= h) { - int m = (l + h) >> 1; - int p = sPtr + m * SegmentIndexSize; - - // read the segment index - read(p, buff); - long sip = getIntLong(buff, 0); - if (ip < sip) { - h = m - 1; - } else { - long eip = getIntLong(buff, 4); - if (ip > eip) { - l = m + 1; - } else { - dataLen = getInt2(buff, 8); - dataPtr = getInt(buff, 10); - break; - } - } - } - - // empty match interception - // System.out.printf("dataLen: %d, dataPtr: %d\n", dataLen, dataPtr); - if (dataPtr < 0) { - return null; - } - - // load and return the region data - final byte[] regionBuff = new byte[dataLen]; - read(dataPtr, regionBuff); - return new String(regionBuff, "utf-8"); - } - - protected void read(int offset, byte[] buffer) throws IOException { - // check the in-memory buffer first - if (contentBuff != null) { - // @TODO: reduce data copying, directly decode the data ? - System.arraycopy(contentBuff, offset, buffer, 0, buffer.length); - return; - } - - // read from the file handle - assert handle != null; - handle.seek(offset); - - this.ioCount++; - int rLen = handle.read(buffer); - if (rLen != buffer.length) { - throw new IOException("incomplete read: read bytes should be " + buffer.length); - } - } - - // --- static cache util function - - public static Header loadHeader(RandomAccessFile handle) throws IOException { - handle.seek(0); - final byte[] buff = new byte[HeaderInfoLength]; - handle.read(buff); - return new Header(buff); - } - - public static Header loadHeaderFromFile(String dbPath) throws IOException { - final RandomAccessFile handle = new RandomAccessFile(dbPath, "r"); - final Header header = loadHeader(handle); - handle.close(); - return header; - } - - public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException { - handle.seek(HeaderInfoLength); - int len = VectorIndexRows * VectorIndexCols * SegmentIndexSize; - final byte[] buff = new byte[len]; - int rLen = handle.read(buff); - if (rLen != len) { - throw new IOException("incomplete read: read bytes should be " + len); - } - - return buff; - } - - public static byte[] loadVectorIndexFromFile(String dbPath) throws IOException { - final RandomAccessFile handle = new RandomAccessFile(dbPath, "r"); - final byte[] vIndex = loadVectorIndex(handle); - handle.close(); - return vIndex; - } - - public static byte[] loadContent(RandomAccessFile handle) throws IOException { - handle.seek(0); - final byte[] buff = new byte[(int) handle.length()]; - int rLen = handle.read(buff); - if (rLen != buff.length) { - throw new IOException("incomplete read: read bytes should be " + buff.length); - } - - return buff; - } - - public static byte[] loadContentFromFile(String dbPath) throws IOException { - final RandomAccessFile handle = new RandomAccessFile(dbPath, "r"); - final byte[] content = loadContent(handle); - handle.close(); - return content; - } - - // --- End cache load util function - - // --- static util method - - /* get an int from a byte array start from the specified offset */ - public static long getIntLong(byte[] b, int offset) { - return ( - ((b[offset++] & 0x000000FFL)) | - ((b[offset++] << 8) & 0x0000FF00L) | - ((b[offset++] << 16) & 0x00FF0000L) | - ((b[offset ] << 24) & 0xFF000000L) - ); - } - - public static int getInt(byte[] b, int offset) { - return ( - ((b[offset++] & 0x000000FF)) | - ((b[offset++] << 8) & 0x0000FF00) | - ((b[offset++] << 16) & 0x00FF0000) | - ((b[offset ] << 24) & 0xFF000000) - ); - } - - public static int getInt2(byte[] b, int offset) { - return ( - (b[offset++] & 0x000000FF) | - (b[offset ] & 0x0000FF00) - ); - } - - /* long int to ip string */ - public static String long2ip( long ip ) - { - return String.valueOf((ip >> 24) & 0xFF) + '.' + - ((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF); - } - - public static final byte[] shiftIndex = {24, 16, 8, 0}; - - /* check the specified ip address */ - public static long checkIP(String ip) throws Exception { - String[] ps = ip.split("\\."); - if (ps.length != 4) { - throw new Exception("invalid ip address `" + ip + "`"); - } - - long ipDst = 0; - for (int i = 0; i < ps.length; i++) { - int val = Integer.parseInt(ps[i]); - if (val > 255) { - throw new Exception("ip part `"+ps[i]+"` should be less then 256"); + public RegionMsg toRegionMsg() { + if (region == null) { + return null; } - ipDst |= ((long) val << shiftIndex[i]); + final String[] ss = region.split("\\|"); + return RegionMsg.builder() + .country(ss[0]) + .region(ss[1]) + .province(ss[2]) + .city(ss[3]) + .isp(ss[4]) + .build(); } - - return ipDst & 0xFFFFFFFFL; } + // Country|Region|Province|City|ISP + // 国家|区域|省份|城市|ISP + @Data + @Builder + class RegionMsg { + private String country; + private String region; + private String province; + private String city; + private String isp; + } } \ No newline at end of file diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/SearcherException.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/SearcherException.java new file mode 100644 index 0000000..e93ce5d --- /dev/null +++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/SearcherException.java @@ -0,0 +1,25 @@ +package org.lionsoul.ip2region.xdb; + +/** + * Searcher RuntimeException + */ +public class SearcherException extends RuntimeException { + public SearcherException() { + } + + public SearcherException(String message) { + super(message); + } + + public SearcherException(String message, Throwable cause) { + super(message, cause); + } + + public SearcherException(Throwable cause) { + super(cause); + } + + public SearcherException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/UtilTest.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/UtilTest.java similarity index 84% rename from binding/java/src/main/java/org/lionsoul/ip2region/UtilTest.java rename to binding/java/src/main/java/org/lionsoul/ip2region/xdb/UtilTest.java index aba7779..14fe249 100644 --- a/binding/java/src/main/java/org/lionsoul/ip2region/UtilTest.java +++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/UtilTest.java @@ -4,17 +4,16 @@ // @Author Lion // @Date 2022/06/23 -package org.lionsoul.ip2region; - -import org.lionsoul.ip2region.xdb.Searcher; +package org.lionsoul.ip2region.xdb; +// for test public class UtilTest { public static void testIP2Long() { String ip = "1.2.3.4"; long ipAddr = 0; try { - ipAddr = Searcher.checkIP(ip); + ipAddr = InternalUtil.ip2long(ip); } catch (Exception e) { System.out.printf("failed to check ip: %s\n", e); return; @@ -25,7 +24,7 @@ public class UtilTest { return; } - String ip2 = Searcher.long2ip(ipAddr); + String ip2 = InternalUtil.long2ip(ipAddr); if (!ip.equals(ip2)) { System.out.print("failed long2ip\n"); return; diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/VectorIndexSearcher.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/VectorIndexSearcher.java new file mode 100644 index 0000000..a920fef --- /dev/null +++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/VectorIndexSearcher.java @@ -0,0 +1,52 @@ +package org.lionsoul.ip2region.xdb; + +import java.io.IOException; +import java.io.RandomAccessFile; + +import static org.lionsoul.ip2region.xdb.InternalUtil.getInt; + +/** + * vector index. + * use the byte[] instead of VectorIndex entry array to keep the minimal memory allocation. + */ +public class VectorIndexSearcher extends FileSearcher { + + private final byte[] vectorIndex; + + public VectorIndexSearcher(String dbFile) { + this(dbFile, null); + } + + public VectorIndexSearcher(String dbFile, byte[] vectorIndex) { + super(dbFile); + try { + this.vectorIndex = vectorIndex != null ? vectorIndex : loadVectorIndexFromFile(dbFile); + } catch (SearcherException | IOException e) { + throw new SearcherException("load dbFile error: " + e.getMessage() + ", file=" + dbFile, e); + } + } + + @Override + protected PointIndex pointIndex(int idx) { + return new PointIndex( + getInt(vectorIndex, idx), + getInt(vectorIndex, idx + 4) + ); + } + + // --- static util function + + private static byte[] loadVectorIndexFromFile(String dbPath) throws IOException { + final byte[] buff; + try (RandomAccessFile handle = new RandomAccessFile(dbPath, "r")) { + handle.seek(HEADER_INFO_LENGTH); + int len = VECTOR_INDEX_ROWS * VECTOR_INDEX_COLS * SEGMENT_INDEX_SIZE; + buff = new byte[len]; + int rLen = handle.read(buff); + if (rLen != len) { + throw new IOException("incomplete read: read bytes should be " + len); + } + } + return buff; + } +}