feat: ref interface Searcher and impl

binding/java/ReadMe.md

@@ -11,140 +11,57 @@
 </dependency>
 ```
 
-### 完全基于文件的查询
+### Example
 
 ```java
+package org.lionsoul.ip2region.example;
+
 import org.lionsoul.ip2region.xdb.Searcher;
-import java.io.*;
+
 import java.util.concurrent.TimeUnit;
 
-public class SearcherTest {
-    public static void main(String[] args) {
-        // 1、创建 searcher 对象
+/**
+ * @see org.lionsoul.ip2region.xdb.Searcher
+ */
+public class SearcherExample {
+
+    public static void main(String[] args) throws Exception {
         String dbPath = "ip2region.xdb file path";
-        Searcher searcher = null;
-        try {
-            searcher = Searcher.newWithFileOnly(dbPath);
-        } catch (IOException e) {
-            System.out.printf("failed to create searcher with `%s`: %s\n", dbPath, e);
-            return;
+
+        // 基于文件查询，单次 search 需要文件 IO，非线程安全，性能最差
+        try (Searcher fileSearcher = Searcher.newWithFileOnly(dbPath)) {
+            searchExample(fileSearcher);
         }
 
-        // 2、查询
-        try {
+        // 基于 VectorIndex 查询，单次 search 需要内存查询 + 文件 IO，非线程安全，性能次之
+        try (Searcher indexSearcher = Searcher.newWithVectorIndex(dbPath)) {
+            searchExample(indexSearcher);
+        }
+
+        // 推荐：基于 Buffer 查询，单次 search 全使用内存查询，线程安全，支持序列化，性能最佳
+        // 可重写 loadFile 方法扩展从其他文件系统(hdfs/s3/oss...)装载 buffer 数据
+        try (Searcher bufferSearcher = Searcher.newWithBuffer(dbPath)) {
+            searchExample(bufferSearcher);
+        }
+
+        // 备注：并发查询时，对于非线程安全实现推荐为每个线程创建一个独立的 searcher 对象单独使用。
+    }
+
+    private static void searchExample(Searcher searcher) {
         String ip = "1.2.3.4";
+        try {
             long sTime = System.nanoTime();
-            String region = searcher.search(ip);
-            long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
-            System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
+            Searcher.Region region = searcher.searchRegion(ip);
+            long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime);
+            System.out.printf("{region: %s, searcher: %s, ioCount: %d, took: %d μs}\n",
+                    region, searcher.getClass().getSimpleName(), region.getIoCount(), cost);
+            System.out.println(region.toRegionMsg());
         } catch (Exception e) {
             System.out.printf("failed to search(%s): %s\n", ip, e);
         }
-
-        // 3、关闭资源
-        searcher.close();
-        
-        // 备注：并发使用，每个线程需要创建一个独立的 searcher 对象单独使用。
     }
 }
-```
 
-### 缓存 `VectorIndex` 索引
-
-我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据，然后全局缓存，每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作，从而加速查询，减少 IO 压力。
-```java
-import org.lionsoul.ip2region.xdb.Searcher;
-import java.io.*;
-import java.util.concurrent.TimeUnit;
-
-public class SearcherTest {
-    public static void main(String[] args) {
-        String dbPath = "ip2region.xdb file path";
-
-        // 1、从 dbPath 中预先加载 VectorIndex 缓存，并且把这个得到的数据作为全局变量，后续反复使用。
-        byte[] vIndex;
-        try {
-            vIndex = Searcher.loadVectorIndexFromFile(dbPath);
-        } catch (Exception e) {
-            System.out.printf("failed to load vector index from `%s`: %s\n", dbPath, e);
-            return;
-        }
-
-        // 2、使用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。
-        Searcher searcher;
-        try {
-            searcher = Searcher.newWithVectorIndex(dbPath, vIndex);
-        } catch (Exception e) {
-            System.out.printf("failed to create vectorIndex cached searcher with `%s`: %s\n", dbPath, e);
-            return;
-        }
-
-        // 3、查询
-        try {
-            String ip = "1.2.3.4";
-            long sTime = System.nanoTime();
-            String region = searcher.search(ip);
-            long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
-            System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
-        } catch (Exception e) {
-            System.out.printf("failed to search(%s): %s\n", ip, e);
-        }
-        
-        // 4、关闭资源
-        searcher.close();
-
-        // 备注：每个线程需要单独创建一个独立的 Searcher 对象，但是都共享全局的制度 vIndex 缓存。
-    }
-}
-```
-
-### 缓存整个 `xdb` 数据
-
-我们也可以预先加载整个 ip2region.xdb 的数据到内存，然后基于这个数据创建查询对象来实现完全基于文件的查询，类似之前的 memory search。
-```java
-import org.lionsoul.ip2region.xdb.Searcher;
-import java.io.*;
-import java.util.concurrent.TimeUnit;
-
-public class SearcherTest {
-    public static void main(String[] args) {
-        String dbPath = "ip2region.xdb file path";
-
-        // 1、从 dbPath 加载整个 xdb 到内存。
-        byte[] cBuff;
-        try {
-            cBuff = Searcher.loadContentFromFile(dbPath);
-        } catch (Exception e) {
-            System.out.printf("failed to load content from `%s`: %s\n", dbPath, e);
-            return;
-        }
-
-        // 2、使用上述的 cBuff 创建一个完全基于内存的查询对象。
-        Searcher searcher;
-        try {
-            searcher = Searcher.newWithBuffer(cBuff);
-        } catch (Exception e) {
-            System.out.printf("failed to create content cached searcher: %s\n", e);
-            return;
-        }
-
-        // 3、查询
-        try {
-            String ip = "1.2.3.4";
-            long sTime = System.nanoTime();
-            String region = searcher.search(ip);
-            long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
-            System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
-        } catch (Exception e) {
-            System.out.printf("failed to search(%s): %s\n", ip, e);
-        }
-        
-        // 4、关闭资源 - 该 searcher 对象可以安全用于并发，等整个服务关闭的时候再关闭 searcher
-        // searcher.close();
-
-        // 备注：并发使用，用整个 xdb 数据缓存创建的查询对象可以安全的用于并发，也就是你可以把这个 searcher 对象做成全局对象去跨线程访问。
-    }
-}
 ```
 
 
@@ -174,7 +91,7 @@ options:
 
 例如：使用默认的 data/ip2region.xdb 文件进行查询测试：
 ```bash
-➜  java git:(v2.0_xdb) ✗ java -jar target/ip2region-2.6.0.jar search --db=../../data/ip2region.xdb
+$ java -jar target/ip2region-2.6.5.jar search --db=../../data/ip2region.xdb
 ip2region xdb searcher test program, cachePolicy: vectorIndex
 type 'quit' to exit
 ip2region>> 1.2.3.4
@@ -182,26 +99,31 @@ ip2region>> 1.2.3.4
 ip2region>>
 ```
 
-输入 ip 即可进行查询测试，也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的查询效果。
+输入 ip 即可进行查询测试，也可以分别设置 `--cache-policy` 来测试三种不同缓存实现的查询效果。
 
 
 # bench 测试
 
 可以通过 `java -jar ip2region-{version}.jar bench` 命令来进行 bench 测试，一方面确保 `xdb` 文件没有错误，一方面可以评估查询性能：
 ```bash
-➜  java git:(v2.0_xdb) ✗ java -jar target/ip2region-2.6.0.jar bench
+java -jar target/ip2region-2.6.5.jar bench
+
 java -jar ip2region-{version}.jar bench [command options]
 options:
  --db string              ip2region binary xdb file path
  --src string             source ip text file path
- --cache-policy string    cache policy: file/vectorIndex/content
+ --cache-policy string    cache policy: file/vectorIndex/buffer
 ```
 
 例如：通过默认的 data/ip2region.xdb 和 data/ip.merge.txt 文件进行 bench 测试：
 ```bash
-➜  java git:(v2.0_xdb) ✗ java -jar target/ip2region-2.6.0.jar bench --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt
-Bench finished, {cachePolicy: vectorIndex, total: 3417955, took: 8s, cost: 2 μs/op}
+$ java -jar target/ip2region-2.6.5.jar bench --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt --cache-policy=buffer
+
+-- 笔者笔记本[mac 2.6 GHz 六核Intel Core i7]测试结果案例
+Bench finished, {cachePolicy: buffer, total: 3417955, took: 2s, ioCount: 0, cost: 0 μs/op}
+Bench finished, {cachePolicy: vectorIndex, total: 3417955, took: 28s, ioCount: 43465554, cost: 8 μs/op}
+Bench finished, {cachePolicy: file, total: 3417955, took: 34s, ioCount: 43465554, cost: 9 μs/op}
 ```
 
-可以通过分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效果。
+可以通过分别设置 `--cache-policy` 来测试三种不同缓存实现的效果。
 @Note: 注意 bench 使用的 src 文件要是生成对应 xdb 文件相同的源文件。

binding/java/pom.xml

@@ -44,14 +44,38 @@
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>1.18.24</version>
+            <scope>provided</scope>
+        </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
-            <version>4.13.1</version>
+            <version>4.13.2</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>net.jcip</groupId>
+            <artifactId>jcip-annotations</artifactId>
+            <version>1.0</version>
+            <scope>provided</scope>
+        </dependency>
     </dependencies>
 
+    <repositories>
+        <repository>
+            <id>aliyun</id>
+            <url>https://maven.aliyun.com/repository/public</url>
+        </repository>
+
+        <repository>
+            <id>repo1</id>
+            <url>https://repo1.maven.org/maven2/</url>
+        </repository>
+    </repositories>
+
     <build>
         <plugins>
             <plugin>
@@ -103,7 +127,7 @@
                         <configuration>
                             <transformers>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                    <mainClass>org.lionsoul.ip2region.SearchTest</mainClass>
+                                    <mainClass>org.lionsoul.ip2region.xdb.SearchTest</mainClass>
                                 </transformer>
                             </transformers>
                         </configuration>

binding/java/src/main/java/org/lionsoul/ip2region/example/SearcherExample.java

@@ -0,0 +1,47 @@
+package org.lionsoul.ip2region.example;
+
+import org.lionsoul.ip2region.xdb.Searcher;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @see org.lionsoul.ip2region.xdb.Searcher
+ */
+public class SearcherExample {
+
+    public static void main(String[] args) throws Exception {
+        String dbPath = "ip2region.xdb file path";
+
+        // 基于文件查询，单次 search 需要文件 IO，非线程安全，性能最差
+        try (Searcher fileSearcher = Searcher.newWithFileOnly(dbPath)) {
+            searchExample(fileSearcher);
+        }
+
+        // 基于 VectorIndex 查询，单次 search 需要内存查询 + 文件 IO，非线程安全，性能次之
+        try (Searcher indexSearcher = Searcher.newWithVectorIndex(dbPath)) {
+            searchExample(indexSearcher);
+        }
+
+        // 推荐：基于 Buffer 查询，单次 search 全使用内存查询，线程安全，支持序列化，性能最佳
+        // 可重写 loadFile 方法扩展从其他文件系统(hdfs/s3/oss...)装载 buffer 数据
+        try (Searcher bufferSearcher = Searcher.newWithBuffer(dbPath)) {
+            searchExample(bufferSearcher);
+        }
+
+        // 备注：并发查询时，对于非线程安全实现推荐为每个线程创建一个独立的 searcher 对象单独使用。
+    }
+
+    private static void searchExample(Searcher searcher) {
+        String ip = "1.2.3.4";
+        try {
+            long sTime = System.nanoTime();
+            Searcher.Region region = searcher.searchRegion(ip);
+            long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime);
+            System.out.printf("{region: %s, searcher: %s, ioCount: %d, took: %d μs}\n",
+                    region, searcher.getClass().getSimpleName(), region.getIoCount(), cost);
+            System.out.println(region.toRegionMsg());
+        } catch (Exception e) {
+            System.out.printf("failed to search(%s): %s\n", ip, e);
+        }
+    }
+}

binding/java/src/main/java/org/lionsoul/ip2region/xdb/AbstractSearcher.java

@@ -0,0 +1,106 @@
+package org.lionsoul.ip2region.xdb;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.lionsoul.ip2region.xdb.InternalUtil.*;
+
+/**
+ * @author Li.Wei by 2022/8/6
+ */
+public abstract class AbstractSearcher implements Searcher {
+    // constant defined copied from the xdb maker
+    protected static final int HEADER_INFO_LENGTH = 256;
+    protected static final int VECTOR_INDEX_ROWS = 256;
+    protected static final int VECTOR_INDEX_COLS = 256;
+    protected static final int VECTOR_INDEX_SIZE = 8;
+    protected static final int SEGMENT_INDEX_SIZE = 14;
+
+    @Deprecated
+    public int getIOCount() {
+        return 0;
+    }
+
+    public Region searchRegion(String ip) {
+        return search0(ip2long(ip));
+    }
+
+    public String search(String ip) {
+        return searchRegion(ip).getRegion();
+    }
+
+    private Region search0(long ip) {
+        int ioCount = 0;
+
+        // locate the segment index block based on the vector index
+        int il0 = (int) ((ip >> 24) & 0xFF);
+        int il1 = (int) ((ip >> 16) & 0xFF);
+        int idx = il0 * VECTOR_INDEX_COLS * VECTOR_INDEX_SIZE + il1 * VECTOR_INDEX_SIZE;
+        // System.out.printf("il0: %d, il1: %d, idx: %d\n", il0, il1, idx);
+        PointIndex pointIndex = this.pointIndex(idx);
+        int sPtr = pointIndex.sPtr, ePtr = pointIndex.ePtr;
+        // System.out.printf("sPtr: %d, ePtr: %d\n", sPtr, ePtr);
+
+        // binary search the segment index block to get the region info
+        final byte[] buff = new byte[SEGMENT_INDEX_SIZE];
+        int dataLen = -1, dataPtr = -1;
+        int l = 0, h = (ePtr - sPtr) / SEGMENT_INDEX_SIZE;
+        while (l <= h) {
+            int m = (l + h) >> 1;
+            int p = sPtr + m * SEGMENT_INDEX_SIZE;
+
+            // read the segment index
+            ioCount += read(p, buff);
+
+            long sip = getIntLong(buff, 0);
+            if (ip < sip) {
+                h = m - 1;
+            } else {
+                long eip = getIntLong(buff, 4);
+                if (ip > eip) {
+                    l = m + 1;
+                } else {
+                    dataLen = getInt2(buff, 8);
+                    dataPtr = getInt(buff, 10);
+                    break;
+                }
+            }
+        }
+
+        // empty match interception
+        // System.out.printf("dataLen: %d, dataPtr: %d\n", dataLen, dataPtr);
+        if (dataPtr < 0) {
+            return Region.builder().ioCount(ioCount).build();
+        }
+
+        // load and return the region data
+        final byte[] regionBuff = new byte[dataLen];
+        ioCount += read(dataPtr, regionBuff);
+        return Region.builder().region(new String(regionBuff, StandardCharsets.UTF_8)).ioCount(ioCount).build();
+    }
+
+    /**
+     * find index start,end point
+     *
+     * @param idx idx
+     * @return PointIndex
+     */
+    protected abstract PointIndex pointIndex(int idx);
+
+    /**
+     * read 2 buffer
+     *
+     * @param offset offset
+     * @param buffer buffer
+     * @return io count
+     */
+    protected abstract int read(int offset, byte[] buffer);
+
+    protected static class PointIndex {
+        int sPtr, ePtr;
+
+        public PointIndex(int sPtr, int ePtr) {
+            this.sPtr = sPtr;
+            this.ePtr = ePtr;
+        }
+    }
+}

binding/java/src/main/java/org/lionsoul/ip2region/xdb/BufferSearcher.java

@@ -0,0 +1,73 @@
+package org.lionsoul.ip2region.xdb;
+
+import net.jcip.annotations.ThreadSafe;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.io.Serializable;
+
+import static org.lionsoul.ip2region.xdb.InternalUtil.getInt;
+
+/**
+ * xdb content buffer, used for in-memory search.
+ */
+@ThreadSafe
+public class BufferSearcher extends AbstractSearcher implements Serializable {
+
+    private final byte[] contentBuff;
+
+    public BufferSearcher(byte[] contentBuff) {
+        this.contentBuff = contentBuff;
+    }
+
+    public BufferSearcher(String dbFile) {
+        try {
+            this.contentBuff = loadFile(dbFile);
+        } catch (IOException e) {
+            throw new SearcherException("load dbFile error: " + e.getMessage() + ", file=" + dbFile, e);
+        }
+    }
+
+    /**
+     * default load local file. override load hdfs/s3/oss...
+     *
+     * @param dbFile dbFile
+     * @return byte array
+     * @throws IOException ex
+     */
+    protected byte[] loadFile(String dbFile) throws IOException {
+        return loadLocalFile(dbFile);
+    }
+
+    @Override
+    protected int read(int offset, byte[] buffer) {
+        System.arraycopy(contentBuff, offset, buffer, 0, buffer.length);
+        return 0;
+    }
+
+    @Override
+    protected PointIndex pointIndex(int idx) {
+        return new PointIndex(
+                getInt(contentBuff, HEADER_INFO_LENGTH + idx),
+                getInt(contentBuff, HEADER_INFO_LENGTH + idx + 4)
+        );
+    }
+
+    @Override
+    public void close() {
+        // nothing
+    }
+
+    private static byte[] loadLocalFile(String dbPath) throws IOException {
+        final byte[] buff;
+        try (RandomAccessFile handle = new RandomAccessFile(dbPath, "r")) {
+            handle.seek(0);
+            buff = new byte[(int) handle.length()];
+            int rLen = handle.read(buff);
+            if (rLen != buff.length) {
+                throw new IOException("incomplete read: read bytes should be " + buff.length);
+            }
+        }
+        return buff;
+    }
+}

binding/java/src/main/java/org/lionsoul/ip2region/xdb/FileSearcher.java

@@ -0,0 +1,52 @@
+package org.lionsoul.ip2region.xdb;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+import static org.lionsoul.ip2region.xdb.InternalUtil.getInt;
+
+/**
+ * random access file handle for file based search.
+ */
+public class FileSearcher extends AbstractSearcher {
+    private final RandomAccessFile handle;
+
+    public FileSearcher(String dbFile) {
+        try {
+            this.handle = new RandomAccessFile(dbFile, "r");
+        } catch (FileNotFoundException e) {
+            throw new SearcherException("load dbFile error: " + e.getMessage() + ", file=" + dbFile, e);
+        }
+    }
+
+    @Override
+    protected int read(int offset, byte[] buffer) {
+        int rLen;
+        try {
+            handle.seek(offset);
+            rLen = handle.read(buffer);
+        } catch (SearcherException | IOException e) {
+            throw new SearcherException("read error", e);
+        }
+        if (rLen != buffer.length) {
+            throw new SearcherException("incomplete read: read bytes should be " + buffer.length);
+        }
+        return 2;
+    }
+
+    @Override
+    protected PointIndex pointIndex(int idx) {
+        final byte[] buff = new byte[VECTOR_INDEX_SIZE];
+        read(HEADER_INFO_LENGTH + idx, buff);
+        return new PointIndex(
+                getInt(buff, 0),
+                getInt(buff, 4)
+        );
+    }
+
+    @Override
+    public void close() throws IOException {
+        this.handle.close();
+    }
+}

binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java

@@ -7,9 +7,8 @@
 package org.lionsoul.ip2region.xdb;
 
 
-import java.awt.image.SampleModel;
-
-public class Header {
+// Internal class
+class Header {
     public final int version;
     public final int indexPolicy;
     public final int createdAt;
@@ -19,15 +18,16 @@ public class Header {
 
     public Header(byte[] buff) {
         assert buff.length >= 16;
-        version = Searcher.getInt2(buff, 0);
-        indexPolicy = Searcher.getInt2(buff, 2);
-        createdAt = Searcher.getInt(buff, 4);
-        startIndexPtr = Searcher.getInt(buff, 8);
-        endIndexPtr = Searcher.getInt(buff, 12);
+        version = InternalUtil.getInt2(buff, 0);
+        indexPolicy = InternalUtil.getInt2(buff, 2);
+        createdAt = InternalUtil.getInt(buff, 4);
+        startIndexPtr = InternalUtil.getInt(buff, 8);
+        endIndexPtr = InternalUtil.getInt(buff, 12);
         buffer = buff;
     }
 
-    @Override public String toString() {
+    @Override
+    public String toString() {
         return "{" +
                 "Version: " + version + ',' +
                 "IndexPolicy: " + indexPolicy + ',' +

binding/java/src/main/java/org/lionsoul/ip2region/xdb/InternalUtil.java

@@ -0,0 +1,58 @@
+package org.lionsoul.ip2region.xdb;
+
+/**
+ * Internal Util
+ */
+public class InternalUtil {
+    private static final byte[] SHIFT_INDEX = {24, 16, 8, 0};
+
+    /* get an int from a byte array start from the specified offset */
+    protected static long getIntLong(byte[] b, int offset) {
+        return (
+                ((b[offset++] & 0x000000FFL)) |
+                        ((b[offset++] << 8) & 0x0000FF00L) |
+                        ((b[offset++] << 16) & 0x00FF0000L) |
+                        ((b[offset] << 24) & 0xFF000000L)
+        );
+    }
+
+    protected static int getInt(byte[] b, int offset) {
+        return (
+                ((b[offset++] & 0x000000FF)) |
+                        ((b[offset++] << 8) & 0x0000FF00) |
+                        ((b[offset++] << 16) & 0x00FF0000) |
+                        ((b[offset] << 24) & 0xFF000000)
+        );
+    }
+
+    protected static int getInt2(byte[] b, int offset) {
+        return (
+                (b[offset++] & 0x000000FF) |
+                        (b[offset] & 0x0000FF00)
+        );
+    }
+
+    /* long int to ip string */
+    public static String long2ip(long ip) {
+        return String.valueOf((ip >> 24) & 0xFF) + '.' +
+                ((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF);
+    }
+
+    /* check the specified ip address */
+    public static long ip2long(String ip) {
+        String[] ps = ip.split("\\.");
+        if (ps.length != 4) {
+            throw new SearcherException("invalid ip address `" + ip + "`");
+        }
+
+        long ipDst = 0;
+        for (int i = 0; i < ps.length; i++) {
+            int val = Integer.parseInt(ps[i]);
+            if (val > 255) {
+                throw new SearcherException("ip part `" + ps[i] + "` should be less then 256");
+            }
+            ipDst |= ((long) val << SHIFT_INDEX[i]);
+        }
+        return ipDst & 0xFFFFFFFFL;
+    }
+}

binding/java/src/main/java/org/lionsoul/ip2region/xdb/SearchTest.java

@@ -4,16 +4,23 @@
 // @Author Lion <chenxin619315@gmail.com>
 // @Date   2022/06/23
 
-package org.lionsoul.ip2region;
+package org.lionsoul.ip2region.xdb;
 
-import org.lionsoul.ip2region.xdb.Searcher;
-
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 import java.util.concurrent.TimeUnit;
 
+// for test
 public class SearchTest {
 
-    public static void printHelp(String[] args) {
+    protected static final String FILE = "file";
+    protected static final String VECTOR_INDEX = "vectorIndex";
+    protected static final String BUFFER = "buffer";
+
+    public static void printHelp() {
         System.out.print("ip2region xdb searcher\n");
         System.out.print("java -jar ip2region-{version}.jar [command] [command options]\n");
         System.out.print("Command: \n");
@@ -21,22 +28,21 @@ public class SearchTest {
         System.out.print("  bench     search bench test\n");
     }
 
-    public static Searcher createSearcher(String dbPath, String cachePolicy) throws IOException {
-        if ("file".equals(cachePolicy)) {
-            return Searcher.newWithFileOnly(dbPath);
-        } else if ("vectorIndex".equals(cachePolicy)) {
-            byte[] vIndex = Searcher.loadVectorIndexFromFile(dbPath);
-            return Searcher.newWithVectorIndex(dbPath, vIndex);
-        } else if ("content".equals(cachePolicy)) {
-            byte[] cBuff = Searcher.loadContentFromFile(dbPath);
-            return Searcher.newWithBuffer(cBuff);
-        } else {
-            throw new IOException("invalid cache policy `" + cachePolicy + "`, options: file/vectorIndex/content");
+    public static AbstractSearcher createSearcher(String dbPath, String cachePolicy) throws IOException {
+        switch (cachePolicy) {
+            case FILE:
+                return (AbstractSearcher) Searcher.newWithFileOnly(dbPath);
+            case VECTOR_INDEX:
+                return (AbstractSearcher) Searcher.newWithVectorIndex(dbPath);
+            case BUFFER:
+                return (AbstractSearcher) Searcher.newWithBuffer(dbPath);
+            default:
+                throw new IOException("invalid cache policy `" + cachePolicy + "`, see help");
         }
     }
 
-    public static void searchTest(String[] args) throws IOException {
-        String dbPath = "", cachePolicy = "vectorIndex";
+    public static void searchTest(String[] args) throws Exception {
+        String dbPath = "", cachePolicy = VECTOR_INDEX;
         for (final String r : args) {
             if (r.length() < 5) {
                 continue;
@@ -74,7 +80,7 @@ public class SearchTest {
         }
 
         Searcher searcher = createSearcher(dbPath, cachePolicy);
-        final BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
+        final BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8));
         System.out.printf("ip2region xdb searcher test program, cachePolicy: %s\ntype 'quit' to exit\n", cachePolicy);
         while (true) {
             System.out.print("ip2region>> ");
@@ -91,9 +97,9 @@ public class SearchTest {
                 double sTime = System.nanoTime();
                 String region = searcher.search(line);
                 long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
-                System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
+                System.out.printf("{region: %s,  took: %d μs}\n", region, cost);
             } catch (Exception e) {
-                System.out.printf("{err: %s, ioCount: %d}\n", e, searcher.getIOCount());
+                System.out.printf("{err: %s}\n", e);
             }
         }
 
@@ -102,8 +108,8 @@ public class SearchTest {
         System.out.println("searcher test program exited, thanks for trying");
     }
 
-    public static void benchTest(String[] args) throws IOException {
-        String dbPath = "", srcPath = "", cachePolicy = "vectorIndex";
+    public static void benchTest(String[] args) throws Exception {
+        String dbPath = "", srcPath = "", cachePolicy = VECTOR_INDEX;
         for (final String r : args) {
             if (r.length() < 5) {
                 continue;
@@ -121,13 +127,17 @@ public class SearchTest {
 
             String key = r.substring(2, sIdx);
             String val = r.substring(sIdx + 1);
-            if ("db".equals(key)) {
+            switch (key) {
+                case "db":
                     dbPath = val;
-            } else if ("src".equals(key)) {
+                    break;
+                case "src":
                     srcPath = val;
-            } else if ("cache-policy".equals(key)) {
+                    break;
+                case "cache-policy":
                     cachePolicy = val;
-            } else {
+                    break;
+                default:
                     System.out.printf("undefined option `%s`\n", r);
                     return;
             }
@@ -138,12 +148,13 @@ public class SearchTest {
             System.out.print("options:\n");
             System.out.print(" --db string              ip2region binary xdb file path\n");
             System.out.print(" --src string             source ip text file path\n");
-            System.out.print(" --cache-policy string    cache policy: file/vectorIndex/content\n");
+            System.out.print(" --cache-policy string    cache policy: " + FILE + "/" + VECTOR_INDEX + "/" + BUFFER + "\n");
+
             return;
         }
 
-        Searcher searcher = createSearcher(dbPath, cachePolicy);
-        long count = 0, costs = 0, tStart = System.nanoTime();
+        AbstractSearcher searcher = createSearcher(dbPath, cachePolicy);
+        long count = 0, costs = 0, ioCount = 0, tStart = System.nanoTime();
         String line;
         final BufferedReader reader = new BufferedReader(new FileReader(srcPath));
         while ((line = reader.readLine()) != null) {
@@ -156,7 +167,7 @@ public class SearchTest {
 
             long sip;
             try {
-                sip = Searcher.checkIP(ps[0]);
+                sip = InternalUtil.ip2long(ps[0]);
             } catch (Exception e) {
                 System.out.printf("check start ip `%s`: %s\n", ps[0], e);
                 return;
@@ -164,7 +175,7 @@ public class SearchTest {
 
             long eip;
             try {
-                eip = Searcher.checkIP(ps[1]);
+                eip = InternalUtil.ip2long(ps[1]);
             } catch (Exception e) {
                 System.out.printf("check end ip `%s`: %s\n", ps[1], e);
                 return;
@@ -178,12 +189,12 @@ public class SearchTest {
             long mip = (sip + eip) >> 1;
             for (final long ip : new long[]{sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) {
                 long sTime = System.nanoTime();
-                String region = searcher.search(ip);
+                Searcher.Region region = searcher.searchRegion(InternalUtil.long2ip(ip));
                 costs += System.nanoTime() - sTime;
-
+                ioCount += region.getIoCount();
                 // check the region info
-                if (!ps[2].equals(region)) {
-                    System.out.printf("failed search(%s) with (%s != %s)\n", Searcher.long2ip(ip), region, ps[2]);
+                if (!ps[2].equals(region.getRegion())) {
+                    System.out.printf("failed search(%s) with (%s != %s)\n", InternalUtil.long2ip(ip), region, ps[2]);
                     return;
                 }
 
@@ -194,31 +205,36 @@ public class SearchTest {
         reader.close();
         searcher.close();
         long took = System.nanoTime() - tStart;
-        System.out.printf("Bench finished, {cachePolicy: %s, total: %d, took: %ds, cost: %d μs/op}\n",
+        System.out.printf("Bench finished, {cachePolicy: %s, total: %d, took: %ds, ioCount: %d, cost: %d μs/op}\n",
                 cachePolicy, count, TimeUnit.NANOSECONDS.toSeconds(took),
+                ioCount,
                 count == 0 ? 0 : TimeUnit.NANOSECONDS.toMicros(costs / count));
     }
 
     public static void main(String[] args) {
         if (args.length < 1) {
-            printHelp(args);
+            printHelp();
             return;
         }
 
-        if ("search".equals(args[0])) {
+        switch (args[0]) {
+            case "search":
                 try {
                     searchTest(args);
-            } catch (IOException e) {
+                } catch (Exception e) {
                     System.out.printf("failed running search test: %s\n", e);
                 }
-        } else if ("bench".equals(args[0])) {
+                break;
+            case "bench":
                 try {
                     benchTest(args);
-            } catch (IOException e) {
+                } catch (Exception e) {
                     System.out.printf("failed running bench test: %s\n", e);
                 }
-        } else {
-            printHelp(args);
+                break;
+            default:
+                printHelp();
+                break;
         }
     }
 

binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java

@@ -9,264 +9,76 @@ package org.lionsoul.ip2region.xdb;
 // @Date   2022/06/23
 
 
-import java.io.IOException;
-import java.io.RandomAccessFile;
+import lombok.Builder;
+import lombok.Data;
 
-public class Searcher {
-    // constant defined copied from the xdb maker
-    public static final int HeaderInfoLength = 256;
-    public static final int VectorIndexRows  = 256;
-    public static final int VectorIndexCols  = 256;
-    public static final int VectorIndexSize  = 8;
-    public static final int SegmentIndexSize = 14;
+/**
+ * @see FileSearcher
+ * @see VectorIndexSearcher
+ * @see BufferSearcher
+ */
+public interface Searcher extends AutoCloseable {
 
-    // random access file handle for file based search
-    private final RandomAccessFile handle;
-
-    private int ioCount = 0;
-
-    // vector index.
-    // use the byte[] instead of VectorIndex entry array to keep
-    // the minimal memory allocation.
-    private final byte[] vectorIndex;
-
-    // xdb content buffer, used for in-memory search
-    private final byte[] contentBuff;
-
-    // --- static method to create searchers
-
-    public static Searcher newWithFileOnly(String dbPath) throws IOException {
-        return new Searcher(dbPath, null, null);
+    // FileSearcher
+    static Searcher newWithFileOnly(String dbPath) {
+        return new FileSearcher(dbPath);
     }
 
-    public static Searcher newWithVectorIndex(String dbPath, byte[] vectorIndex) throws IOException {
-        return new Searcher(dbPath, vectorIndex, null);
+    // IndexSearcher
+    static Searcher newWithVectorIndex(String dbPath) {
+        return new VectorIndexSearcher(dbPath);
     }
 
-    public static Searcher newWithBuffer(byte[] cBuff) throws IOException {
-        return new Searcher(null, null, cBuff);
+    // IndexSearcher
+    static Searcher newWithVectorIndex(String dbPath, byte[] vectorIndex) {
+        return new VectorIndexSearcher(dbPath, vectorIndex);
     }
 
-    // --- End of creator
-
-    public Searcher(String dbFile, byte[] vectorIndex, byte[] cBuff) throws IOException {
-        if (cBuff != null) {
-            this.handle = null;
-            this.vectorIndex = null;
-            this.contentBuff = cBuff;
-        } else {
-            this.handle = new RandomAccessFile(dbFile, "r");
-            this.vectorIndex = vectorIndex;
-            this.contentBuff = null;
-        }
+    // BufferSearcher
+    static Searcher newWithBuffer(String dbPath) {
+        return new BufferSearcher(dbPath);
     }
 
-    public void close() throws IOException {
-        if (this.handle != null) {
-            this.handle.close();
-        }
+    // BufferSearcher
+    static Searcher newWithBuffer(byte[] cBuff) {
+        return new BufferSearcher(cBuff);
     }
 
-    public int getIOCount() {
-        return ioCount;
-    }
+    String search(String ip);
 
-    public String search(String ipStr) throws Exception {
-        long ip = checkIP(ipStr);
-        return search(ip);
-    }
+    Region searchRegion(String ip);
 
-    public String search(long ip) throws IOException {
-        // reset the global counter
-        this.ioCount = 0;
+    @Data
+    @Builder
+    class Region {
+        private int ioCount;
+        private String region;
 
-        // locate the segment index block based on the vector index
-        int sPtr = 0, ePtr = 0;
-        int il0 = (int) ((ip >> 24) & 0xFF);
-        int il1 = (int) ((ip >> 16) & 0xFF);
-        int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
-        // System.out.printf("il0: %d, il1: %d, idx: %d\n", il0, il1, idx);
-        if (vectorIndex != null) {
-            sPtr = getInt(vectorIndex, idx);
-            ePtr = getInt(vectorIndex, idx + 4);
-        } else if (contentBuff != null) {
-            sPtr = getInt(contentBuff, HeaderInfoLength + idx);
-            ePtr = getInt(contentBuff, HeaderInfoLength + idx + 4);
-        } else {
-            final byte[] buff = new byte[VectorIndexSize];
-            read(HeaderInfoLength + idx, buff);
-            sPtr = getInt(buff, 0);
-            ePtr = getInt(buff, 4);
-        }
-
-        // System.out.printf("sPtr: %d, ePtr: %d\n", sPtr, ePtr);
-
-        // binary search the segment index block to get the region info
-        final byte[] buff = new byte[SegmentIndexSize];
-        int dataLen = -1, dataPtr = -1;
-        int l = 0, h = (ePtr - sPtr) / SegmentIndexSize;
-        while (l <= h) {
-            int m = (l + h) >> 1;
-            int p = sPtr + m * SegmentIndexSize;
-
-            // read the segment index
-            read(p, buff);
-            long sip = getIntLong(buff, 0);
-            if (ip < sip) {
-                h = m - 1;
-            } else {
-                long eip = getIntLong(buff, 4);
-                if (ip > eip) {
-                    l = m + 1;
-                } else {
-                    dataLen = getInt2(buff, 8);
-                    dataPtr = getInt(buff, 10);
-                    break;
-                }
-            }
-        }
-
-        // empty match interception
-        // System.out.printf("dataLen: %d, dataPtr: %d\n", dataLen, dataPtr);
-        if (dataPtr < 0) {
+        public RegionMsg toRegionMsg() {
+            if (region == null) {
                 return null;
             }
 
-        // load and return the region data
-        final byte[] regionBuff = new byte[dataLen];
-        read(dataPtr, regionBuff);
-        return new String(regionBuff, "utf-8");
-    }
-
-    protected void read(int offset, byte[] buffer) throws IOException {
-        // check the in-memory buffer first
-        if (contentBuff != null) {
-            // @TODO: reduce data copying, directly decode the data ?
-            System.arraycopy(contentBuff, offset, buffer, 0, buffer.length);
-            return;
-        }
-
-        // read from the file handle
-        assert handle != null;
-        handle.seek(offset);
-
-        this.ioCount++;
-        int rLen = handle.read(buffer);
-        if (rLen != buffer.length) {
-            throw new IOException("incomplete read: read bytes should be " + buffer.length);
+            final String[] ss = region.split("\\|");
+            return RegionMsg.builder()
+                    .country(ss[0])
+                    .region(ss[1])
+                    .province(ss[2])
+                    .city(ss[3])
+                    .isp(ss[4])
+                    .build();
         }
     }
 
-    // --- static cache util function
-
-    public static Header loadHeader(RandomAccessFile handle) throws IOException {
-        handle.seek(0);
-        final byte[] buff = new byte[HeaderInfoLength];
-        handle.read(buff);
-        return new Header(buff);
+    // Country|Region|Province|City|ISP
+    // 国家|区域|省份|城市|ISP
+    @Data
+    @Builder
+    class RegionMsg {
+        private String country;
+        private String region;
+        private String province;
+        private String city;
+        private String isp;
     }
-
-    public static Header loadHeaderFromFile(String dbPath) throws IOException {
-        final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
-        final Header header = loadHeader(handle);
-        handle.close();
-        return header;
-    }
-
-    public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException {
-        handle.seek(HeaderInfoLength);
-        int len = VectorIndexRows * VectorIndexCols * SegmentIndexSize;
-        final byte[] buff = new byte[len];
-        int rLen = handle.read(buff);
-        if (rLen != len) {
-            throw new IOException("incomplete read: read bytes should be " + len);
-        }
-
-        return buff;
-    }
-
-    public static byte[] loadVectorIndexFromFile(String dbPath) throws IOException {
-        final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
-        final byte[] vIndex = loadVectorIndex(handle);
-        handle.close();
-        return vIndex;
-    }
-
-    public static byte[] loadContent(RandomAccessFile handle) throws IOException {
-        handle.seek(0);
-        final byte[] buff =  new byte[(int) handle.length()];
-        int rLen = handle.read(buff);
-        if (rLen != buff.length) {
-            throw new IOException("incomplete read: read bytes should be " + buff.length);
-        }
-
-        return buff;
-    }
-
-    public static byte[] loadContentFromFile(String dbPath) throws IOException {
-        final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
-        final byte[] content = loadContent(handle);
-        handle.close();
-        return content;
-    }
-
-    // --- End cache load util function
-
-    // --- static util method
-
-    /* get an int from a byte array start from the specified offset */
-    public static long getIntLong(byte[] b, int offset) {
-        return (
-            ((b[offset++] & 0x000000FFL)) |
-            ((b[offset++] <<  8) & 0x0000FF00L) |
-            ((b[offset++] << 16) & 0x00FF0000L) |
-            ((b[offset  ] << 24) & 0xFF000000L)
-        );
-    }
-
-    public static int getInt(byte[] b, int offset) {
-        return (
-            ((b[offset++] & 0x000000FF)) |
-            ((b[offset++] <<  8) & 0x0000FF00) |
-            ((b[offset++] << 16) & 0x00FF0000) |
-            ((b[offset  ] << 24) & 0xFF000000)
-        );
-    }
-
-    public static int getInt2(byte[] b, int offset) {
-        return (
-            (b[offset++] & 0x000000FF) |
-            (b[offset  ] & 0x0000FF00)
-        );
-    }
-
-    /* long int to ip string */
-    public static String long2ip( long ip )
-    {
-        return String.valueOf((ip >> 24) & 0xFF) + '.' +
-            ((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF);
-    }
-
-    public static final byte[] shiftIndex = {24, 16, 8, 0};
-
-    /* check the specified ip address */
-    public static long checkIP(String ip) throws Exception {
-        String[] ps = ip.split("\\.");
-        if (ps.length != 4) {
-            throw new Exception("invalid ip address `" + ip + "`");
-        }
-
-        long ipDst = 0;
-        for (int i = 0; i < ps.length; i++) {
-            int val = Integer.parseInt(ps[i]);
-            if (val > 255) {
-                throw new Exception("ip part `"+ps[i]+"` should be less then 256");
-            }
-
-            ipDst |= ((long) val << shiftIndex[i]);
-        }
-
-        return ipDst & 0xFFFFFFFFL;
-    }
-
 }

binding/java/src/main/java/org/lionsoul/ip2region/xdb/SearcherException.java

@@ -0,0 +1,25 @@
+package org.lionsoul.ip2region.xdb;
+
+/**
+ * Searcher RuntimeException
+ */
+public class SearcherException extends RuntimeException {
+    public SearcherException() {
+    }
+
+    public SearcherException(String message) {
+        super(message);
+    }
+
+    public SearcherException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public SearcherException(Throwable cause) {
+        super(cause);
+    }
+
+    public SearcherException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}

binding/java/src/main/java/org/lionsoul/ip2region/xdb/UtilTest.java

@@ -4,17 +4,16 @@
 // @Author Lion <chenxin619315@gmail.com>
 // @Date   2022/06/23
 
-package org.lionsoul.ip2region;
-
-import org.lionsoul.ip2region.xdb.Searcher;
+package org.lionsoul.ip2region.xdb;
 
+// for test
 public class UtilTest {
 
     public static void testIP2Long() {
         String ip = "1.2.3.4";
         long ipAddr = 0;
         try {
-            ipAddr = Searcher.checkIP(ip);
+            ipAddr = InternalUtil.ip2long(ip);
         } catch (Exception e) {
             System.out.printf("failed to check ip: %s\n", e);
             return;
@@ -25,7 +24,7 @@ public class UtilTest {
             return;
         }
 
-        String ip2 = Searcher.long2ip(ipAddr);
+        String ip2 = InternalUtil.long2ip(ipAddr);
         if (!ip.equals(ip2)) {
             System.out.print("failed long2ip\n");
             return;

binding/java/src/main/java/org/lionsoul/ip2region/xdb/VectorIndexSearcher.java

@@ -0,0 +1,52 @@
+package org.lionsoul.ip2region.xdb;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+import static org.lionsoul.ip2region.xdb.InternalUtil.getInt;
+
+/**
+ * vector index.
+ * use the byte[] instead of VectorIndex entry array to keep the minimal memory allocation.
+ */
+public class VectorIndexSearcher extends FileSearcher {
+
+    private final byte[] vectorIndex;
+
+    public VectorIndexSearcher(String dbFile) {
+        this(dbFile, null);
+    }
+
+    public VectorIndexSearcher(String dbFile, byte[] vectorIndex) {
+        super(dbFile);
+        try {
+            this.vectorIndex = vectorIndex != null ? vectorIndex : loadVectorIndexFromFile(dbFile);
+        } catch (SearcherException | IOException e) {
+            throw new SearcherException("load dbFile error: " + e.getMessage() + ", file=" + dbFile, e);
+        }
+    }
+
+    @Override
+    protected PointIndex pointIndex(int idx) {
+        return new PointIndex(
+                getInt(vectorIndex, idx),
+                getInt(vectorIndex, idx + 4)
+        );
+    }
+
+    // --- static util function
+
+    private static byte[] loadVectorIndexFromFile(String dbPath) throws IOException {
+        final byte[] buff;
+        try (RandomAccessFile handle = new RandomAccessFile(dbPath, "r")) {
+            handle.seek(HEADER_INFO_LENGTH);
+            int len = VECTOR_INDEX_ROWS * VECTOR_INDEX_COLS * SEGMENT_INDEX_SIZE;
+            buff = new byte[len];
+            int rLen = handle.read(buff);
+            if (rLen != len) {
+                throw new IOException("incomplete read: read bytes should be " + len);
+            }
+        }
+        return buff;
+    }
+}