Merge pull request #411 from lionsoul2014/fr_java_fix_slice_bytes

Fr java fixed slice bytes

binding/java/README.md

@@ -7,7 +7,7 @@
 <dependency>
     <groupId>org.lionsoul</groupId>
     <artifactId>ip2region</artifactId>
-    <version>3.3.2</version>
+    <version>3.3.3</version>
 </dependency>
 ```
 
@@ -58,8 +58,7 @@ ip2Region.close();
 final Ip2Region ip2region = Ip2Region.create(v4Config, v6Config, true);
 ```
 4. 如果配置设置的缓存策略为 `Config.BufferCache` 即 `全内存缓存` 则默认会使用单实例的内存查询器，该实现天生并发安全，此时通过 `setSearchers` 指定的查询器数量无效。
-5. 如果使用的是全内存缓存查询且在加载 xdb 二进制内容的时候提示 `OOM`，请参考 [sliceBytes设置](#slicebytes) 然后通过 `ConfigBuilder.setCacheSliceBytes(int)` 设置一个合适的值来避免 OOM。
+5. 如果 `ip2region` 查询器在提供服务期间，调用 close 默认会最大等待 10 秒钟来等待尽量多的查询器归还。
-6. 如果 `ip2region` 查询器在提供服务期间，调用 close 默认会最大等待 10 秒钟来等待尽量多的查询器归还。
 
 
 ### 关于查询 API
@@ -245,10 +244,10 @@ public class SearcherTest {
 
 ### sliceBytes
 
-sliceBytes 表示 xdb 全内存缓存时 `LongByteArray` 类内部维护的 `List<byte[]> buffs` 集合的分片内存的大小，这个值的最大值也是默认值为 `Searcher.MAX_WRITE_BYTES`，取值的核心是为了减少 `buffs` 的长度， 最小值为 1，buffs 长度越小越好，意味着查询过程中的寻址操作的 buffs 遍历操作越少，该值的设置原则如下：
+sliceBytes 表示 xdb 全内存缓存时 `LongByteArray` 类内部维护的 `List<byte[]> buffs` 集合的分片内存的大小，默认值为 `Searcher.DEFAULT_SLICE_BYTES` = `50MiB`，这个值的最大允许值为 `Searcher.MAX_WRITE_BYTES` = `0x7ffff000`，关于该取值的来源可以参考作者博客文章：[https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA](https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA)。
-1. 默认为 `Searcher.MAX_WRITE_BYTES`，也就是 `0x7ffff000`，关于该取值的来源可以参考作者博客文章：[https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA](https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA)。
+1. 从 `3.3.3` 版本开始 `LongByteArray` 实现了固定分片尺寸支持，可以通过简单的计算快速的完成 `offset` 定位的从而实现 `slice` 或者 `copy` 操作。
-2. 如果 xdb 文件的字节数小于 `Searcher.MAX_WRITE_BYTES`，则 sliceBytes 设置为该 xdb 文件的字节数即可，如果大于 `Searcher.MAX_WRITE_BYTES` 则使用默认值即可。Searcher 的 `loadContent` 或者 `loadContentFromFile` 方法默认都是按照这个原则来自动设置 sliceBytes 的值，唯独 `loadContentFromInputStream` 系列方法因为不方便获取流的大小使用的是默认最大值，因此可以按照上述原则通过调用 `loadContentFromInputStream(InputStream, int)` 手动设置合理的值，或者设置 JVM 的内存限制避免运行时的 OOM 错误。
+2. 从计算速度来说 sliceBytes 越大 buffs 的长度越小，计算耗时越小，不过有了固定 sliceBytes 实现这个差距完全可以忽略，所以建议保持默认值为 `50MiB` 即可，也不会出现之前弹性分片尺寸可能导致的 OOM 问题。
-3. 如果 sliceBytes 设置的值小于甚至远远小于 xdb 文件的字节数则会增加查询过程中的寻址遍历操作从而减慢查询，其他无任何影响，随着 IPv6 的普及后期的 xdb 文件大小可能几个G甚至10G+，所以默认 sliceBytes 取的最大值也是为了默认总是能取得最佳的运行效率。
+
 
 
 # 编译测试程序

binding/java/pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>org.lionsoul</groupId>
     <artifactId>ip2region</artifactId>
-    <version>3.3.2</version>
+    <version>3.3.3</version>
     <packaging>jar</packaging>
 
     <name>ip2region</name>

binding/java/src/main/java/org/lionsoul/ip2region/SearcherTest.java

@@ -239,6 +239,9 @@ public class SearcherTest {
                 return;
             }
 
+            // mark the start time
+            long sTime = System.nanoTime();
+
             byte[] sip;
             try {
                 sip = Util.parseIP(ps[0]);
@@ -264,9 +267,7 @@ public class SearcherTest {
             }
 
             for (final byte[] ip : new byte[][]{sip, eip}) {
-                long sTime = System.nanoTime();
                 String region = searcher.search(ip);
-                costs += System.nanoTime() - sTime;
 
                 // check the region info
                 if (!ps[2].equals(region)) {
@@ -277,6 +278,8 @@ public class SearcherTest {
 
                 count++;
             }
+
+            costs += System.nanoTime() - sTime;
         }
 
         reader.close();

binding/java/src/main/java/org/lionsoul/ip2region/xdb/LongByteArray.java

@@ -4,6 +4,8 @@
 
 package org.lionsoul.ip2region.xdb;
 
+import java.io.IOException;
+
 // xdb byte buffer which used to instead of the byte array
 // when the size of the xdb file is greater than 2^32 << 2;
 // xdb file v4 is designed to be a maximum of 2^32 bytes in size.
@@ -14,21 +16,44 @@ import java.util.ArrayList;
 import java.util.List;
 
 public class LongByteArray {
+
+    // slice bytes
+    // if it is greater than the 0 we will use the fixed slice bytes
+    // or we use the dynamic slice bytes.
+    private final int sliceBytes;
+
+    // when EOF is true means we cannot call the #append anymore.
+    // for fixed slice bytes only.
+    private boolean _eof = false;
+
     // byte buffer list
     private final List<byte[]> buffs = new ArrayList<byte[]>();
     private long length;
 
     public LongByteArray() {
         this.length = 0;
+        this.sliceBytes = -1;
     }
 
-    public LongByteArray(byte[] buff) {
+    public LongByteArray(int sliceBytes) {
-        buffs.add(buff);
+        assert sliceBytes != 0;
-        length = buff.length;
+        assert sliceBytes <= Searcher.MAX_WRITE_BYTES;
+        this.sliceBytes = sliceBytes;
     }
 
     // append new buffer
-    public void append(final byte[] buffer) {
+    public void append(final byte[] buffer) throws IOException{
+        // check and assert the slice bytes
+        if (sliceBytes > 0) {
+            if (_eof) {
+                throw new IOException("buffer array closed (EOF=true)");
+            } else if (buffer.length != sliceBytes) {
+                // mark the buffer array as closed
+                // since the last buffer block bytes is not equal to the expected #sliceBytes
+                _eof = true;
+            }
+        }
+
         buffs.add(buffer);
         length += buffer.length;
     }
@@ -44,6 +69,13 @@ public class LongByteArray {
     // internal method to determine the position of the specified offset
     private Position determinate(final long offset) {
         int index = 0, position = 0, buffLen = buffs.size();
+        if (sliceBytes > 0) {
+            // simply some math calcs to determine the offset
+            index = (int) (offset / sliceBytes);
+            position = (int) (offset - (index * sliceBytes));
+            // position = (int) (offset % sliceBytes);
+        } else {
+            // loop the buffer to determine the offset
             long curIndex = 0;
             for (index = 0; index < buffLen; index++) {
                 final byte[] buff = buffs.get(index);
@@ -56,6 +88,7 @@ public class LongByteArray {
                 position = (int) (offset - curIndex);
                 break;
             }
+        }
 
         return new Position(index, position);
     }

binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java

@@ -25,11 +25,15 @@ public class Searcher {
     public static final int VectorIndexCols  = 256;
     public static final int VectorIndexSize  = 8;
 
+    // maximum slice bytes for dynamic buffer array.
     // Linux max write / read bytes.
     // Check https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA 
     //  to get to know why we default to this value.
     public static final int MAX_WRITE_BYTES = 0x7ffff000;
 
+    // default slice bytes (50 MiB) for fixed buffer array.
+    public static final int DEFAULT_SLICE_BYTES = 50 * 1024 * 1024;
+
     // ip version
     private final Version version;
 
@@ -259,14 +263,14 @@ public class Searcher {
     // -- load xdb buffer with random access file handle
     
     public static LongByteArray loadContent(RandomAccessFile handle) throws IOException {
-        return loadContent(handle, MAX_WRITE_BYTES);
+        return loadContent(handle, DEFAULT_SLICE_BYTES);
     }
 
     public static LongByteArray loadContent(RandomAccessFile handle, final int sliceBytes) throws IOException {
         handle.seek(0);
         // check the length and do the buff load
         long toRead = handle.length();
-        final LongByteArray byteArray = new LongByteArray();
+        final LongByteArray byteArray = new LongByteArray(sliceBytes);
         while (toRead > 0) {
             final byte[] buff = new byte[(int) Math.min(toRead, sliceBytes)];
             final int rLen = handle.read(buff);
@@ -284,7 +288,7 @@ public class Searcher {
     // -- load xdb buffer with xdb file object
 
     public static LongByteArray loadContentFromFile(File xdbFile) throws IOException {
-        return loadContentFromFile(xdbFile, MAX_WRITE_BYTES);
+        return loadContentFromFile(xdbFile, DEFAULT_SLICE_BYTES);
     }
 
     public static LongByteArray loadContentFromFile(File xdbFile, final int sliceBytes) throws IOException {
@@ -297,7 +301,7 @@ public class Searcher {
     // -- load xdb buffer with xdb file path
 
     public static LongByteArray loadContentFromFile(String xdbPath) throws IOException {
-        return loadContentFromFile(xdbPath, MAX_WRITE_BYTES);
+        return loadContentFromFile(xdbPath, DEFAULT_SLICE_BYTES);
     }
 
     public static LongByteArray loadContentFromFile(String xdbPath, final int sliceBytes) throws IOException {
@@ -307,11 +311,11 @@ public class Searcher {
     // load xdb buffer from input stream
 
     public static LongByteArray loadContentFromInputStream(InputStream is) throws IOException {
-        return loadContentFromInputStream(is, MAX_WRITE_BYTES);
+        return loadContentFromInputStream(is, DEFAULT_SLICE_BYTES);
     }
 
     public static LongByteArray loadContentFromInputStream(InputStream is, final int sliceBytes) throws IOException {
-        final LongByteArray byteArray = new LongByteArray();
+        final LongByteArray byteArray = new LongByteArray(sliceBytes);
         while (true) {
             boolean done = false;
 

binding/java/src/test/java/org/lionsoul/ip2region/xdb/BufferTest.java

@@ -1,6 +1,8 @@
 package org.lionsoul.ip2region.xdb;
 
+import java.io.File;
 import java.io.FileInputStream;
+import java.io.IOException;
 import java.security.CodeSource;
 
 import org.junit.Test;
@@ -19,6 +21,8 @@ public class BufferTest {
         }
     }
 
+    // --- v4
+
     @Test
     public void testV4InputStreamBuffer() throws Exception {
         final LongByteArray cBuffer = Searcher.loadContentFromInputStream(
@@ -27,6 +31,48 @@ public class BufferTest {
         log.debugf("cBuffer->{length:%d, size:%d}", cBuffer.length(), cBuffer.size());
     }
 
+    @Test
+    public void testV4FixedBuffer() throws Exception {
+        final LongByteArray cBuffer = Searcher.loadContentFromFile(
+            new File(getDataPath("ip2region_v4.xdb")), 2 * 1024 * 1024
+        );
+        final Header header = Searcher.loadHeaderFromBuffer(cBuffer);
+        log.debugf("cBuffer->{length:%d, size:%d}", cBuffer.length(), cBuffer.size());
+        log.debugf("Header->%s", header);
+    }
+
+    @Test
+    public void testV4BufferAssert() throws Exception {
+        final LongByteArray m2Bufer = Searcher.loadContentFromFile(
+            new File(getDataPath("ip2region_v4.xdb")), 2 * 1024 * 1024
+        );
+        final LongByteArray m5Bufer = Searcher.loadContentFromFile(
+            new File(getDataPath("ip2region_v4.xdb")), 5 * 1024 * 1024
+        );
+
+        final int[] offsets = new int[]{0, 10, 512, 1024, 39672, 1024 * 1024 * 2};
+        for (int idx : offsets) {
+            final long m2Val = m2Bufer.getUint32(idx);
+            final long m5Val = m5Bufer.getUint32(idx);
+            log.debugf("m2Buffer[%8d:4]: %10d, m5Buffer[%8d:4]: %10d, equals ? %s", idx, m2Val, idx, m5Val, m2Val == m5Val ? "true" : "false");
+        }
+    }
+
+    @Test
+    public void testV4BufferEOF() throws IOException {
+        final LongByteArray buffer = Searcher.loadContentFromFile(
+            new File(getDataPath("ip2region_v4.xdb")), 2 * 1024 * 1024
+        );
+
+        try {
+            buffer.append(new byte[1024]);
+        } catch (IOException e) {
+            log.debugf("failed to append: %s", e.getMessage());
+        }
+    }
+
+    // --- v6
+
     @Test
     public void testV6InputStreamBuffer() throws Exception {
         final LongByteArray cBuffer = Searcher.loadContentFromInputStream(
@@ -35,4 +81,14 @@ public class BufferTest {
         log.debugf("cBuffer->{length:%d, size:%d}", cBuffer.length(), cBuffer.size());
     }
 
+    @Test
+    public void testV6FixedBuffer() throws Exception {
+        final LongByteArray cBuffer = Searcher.loadContentFromFile(
+            new File(getDataPath("ip2region_v6.xdb")), 5 * 1024 * 1024
+        );
+        final Header header = Searcher.loadHeaderFromBuffer(cBuffer);
+        log.debugf("cBuffer->{length:%d, size:%d}", cBuffer.length(), cBuffer.size());
+        log.debugf("Header->%s", header);
+    }
+
 }