diff --git a/binding/java/README.md b/binding/java/README.md
index f63e438..6b77a4e 100644
--- a/binding/java/README.md
+++ b/binding/java/README.md
@@ -7,7 +7,7 @@
org.lionsoul
ip2region
- 3.3.2
+ 3.3.3
```
@@ -58,8 +58,7 @@ ip2Region.close();
final Ip2Region ip2region = Ip2Region.create(v4Config, v6Config, true);
```
4. 如果配置设置的缓存策略为 `Config.BufferCache` 即 `全内存缓存` 则默认会使用单实例的内存查询器，该实现天生并发安全，此时通过 `setSearchers` 指定的查询器数量无效。
-5. 如果使用的是全内存缓存查询且在加载 xdb 二进制内容的时候提示 `OOM`，请参考 [sliceBytes设置](#slicebytes) 然后通过 `ConfigBuilder.setCacheSliceBytes(int)` 设置一个合适的值来避免 OOM。
-6. 如果 `ip2region` 查询器在提供服务期间，调用 close 默认会最大等待 10 秒钟来等待尽量多的查询器归还。
+5. 如果 `ip2region` 查询器在提供服务期间，调用 close 默认会最大等待 10 秒钟来等待尽量多的查询器归还。
### 关于查询 API
@@ -245,10 +244,10 @@ public class SearcherTest {
### sliceBytes
-sliceBytes 表示 xdb 全内存缓存时 `LongByteArray` 类内部维护的 `List buffs` 集合的分片内存的大小，这个值的最大值也是默认值为 `Searcher.MAX_WRITE_BYTES`，取值的核心是为了减少 `buffs` 的长度， 最小值为 1，buffs 长度越小越好，意味着查询过程中的寻址操作的 buffs 遍历操作越少，该值的设置原则如下：
-1. 默认为 `Searcher.MAX_WRITE_BYTES`，也就是 `0x7ffff000`，关于该取值的来源可以参考作者博客文章：[https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA](https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA)。
-2. 如果 xdb 文件的字节数小于 `Searcher.MAX_WRITE_BYTES`，则 sliceBytes 设置为该 xdb 文件的字节数即可，如果大于 `Searcher.MAX_WRITE_BYTES` 则使用默认值即可。Searcher 的 `loadContent` 或者 `loadContentFromFile` 方法默认都是按照这个原则来自动设置 sliceBytes 的值，唯独 `loadContentFromInputStream` 系列方法因为不方便获取流的大小使用的是默认最大值，因此可以按照上述原则通过调用 `loadContentFromInputStream(InputStream, int)` 手动设置合理的值，或者设置 JVM 的内存限制避免运行时的 OOM 错误。
-3. 如果 sliceBytes 设置的值小于甚至远远小于 xdb 文件的字节数则会增加查询过程中的寻址遍历操作从而减慢查询，其他无任何影响，随着 IPv6 的普及后期的 xdb 文件大小可能几个G甚至10G+，所以默认 sliceBytes 取的最大值也是为了默认总是能取得最佳的运行效率。
+sliceBytes 表示 xdb 全内存缓存时 `LongByteArray` 类内部维护的 `List buffs` 集合的分片内存的大小，默认值为 `Searcher.DEFAULT_SLICE_BYTES` = `50MiB`，这个值的最大允许值为 `Searcher.MAX_WRITE_BYTES` = `0x7ffff000`，关于该取值的来源可以参考作者博客文章：[https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA](https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA)。
+1. 从 `3.3.3` 版本开始 `LongByteArray` 实现了固定分片尺寸支持，可以通过简单的计算快速的完成 `offset` 定位的从而实现 `slice` 或者 `copy` 操作。
+2. 从计算速度来说 sliceBytes 越大 buffs 的长度越小，计算耗时越小，不过有了固定 sliceBytes 实现这个差距完全可以忽略，所以建议保持默认值为 `50MiB` 即可，也不会出现之前弹性分片尺寸可能导致的 OOM 问题。
+
# 编译测试程序
diff --git a/binding/java/pom.xml b/binding/java/pom.xml
index 6929481..7b99943 100644
--- a/binding/java/pom.xml
+++ b/binding/java/pom.xml
@@ -4,7 +4,7 @@
org.lionsoul
ip2region
- 3.3.2
+ 3.3.3
jar
ip2region
diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/SearcherTest.java b/binding/java/src/main/java/org/lionsoul/ip2region/SearcherTest.java
index 33506f9..d0a28ce 100644
--- a/binding/java/src/main/java/org/lionsoul/ip2region/SearcherTest.java
+++ b/binding/java/src/main/java/org/lionsoul/ip2region/SearcherTest.java
@@ -239,6 +239,9 @@ public class SearcherTest {
return;
}
+ // mark the start time
+ long sTime = System.nanoTime();
+
byte[] sip;
try {
sip = Util.parseIP(ps[0]);
@@ -264,9 +267,7 @@ public class SearcherTest {
}
for (final byte[] ip : new byte[][]{sip, eip}) {
- long sTime = System.nanoTime();
String region = searcher.search(ip);
- costs += System.nanoTime() - sTime;
// check the region info
if (!ps[2].equals(region)) {
@@ -277,6 +278,8 @@ public class SearcherTest {
count++;
}
+
+ costs += System.nanoTime() - sTime;
}
reader.close();
diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/LongByteArray.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/LongByteArray.java
index 6d75672..8050250 100644
--- a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/LongByteArray.java
+++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/LongByteArray.java
@@ -4,6 +4,8 @@
package org.lionsoul.ip2region.xdb;
+import java.io.IOException;
+
// xdb byte buffer which used to instead of the byte array
// when the size of the xdb file is greater than 2^32 << 2;
// xdb file v4 is designed to be a maximum of 2^32 bytes in size.
@@ -14,21 +16,44 @@ import java.util.ArrayList;
import java.util.List;
public class LongByteArray {
+
+ // slice bytes
+ // if it is greater than the 0 we will use the fixed slice bytes
+ // or we use the dynamic slice bytes.
+ private final int sliceBytes;
+
+ // when EOF is true means we cannot call the #append anymore.
+ // for fixed slice bytes only.
+ private boolean _eof = false;
+
// byte buffer list
private final List buffs = new ArrayList();
private long length;
public LongByteArray() {
this.length = 0;
+ this.sliceBytes = -1;
}
- public LongByteArray(byte[] buff) {
- buffs.add(buff);
- length = buff.length;
+ public LongByteArray(int sliceBytes) {
+ assert sliceBytes != 0;
+ assert sliceBytes <= Searcher.MAX_WRITE_BYTES;
+ this.sliceBytes = sliceBytes;
}
// append new buffer
- public void append(final byte[] buffer) {
+ public void append(final byte[] buffer) throws IOException{
+ // check and assert the slice bytes
+ if (sliceBytes > 0) {
+ if (_eof) {
+ throw new IOException("buffer array closed (EOF=true)");
+ } else if (buffer.length != sliceBytes) {
+ // mark the buffer array as closed
+ // since the last buffer block bytes is not equal to the expected #sliceBytes
+ _eof = true;
+ }
+ }
+
buffs.add(buffer);
length += buffer.length;
}
@@ -44,17 +69,25 @@ public class LongByteArray {
// internal method to determine the position of the specified offset
private Position determinate(final long offset) {
int index = 0, position = 0, buffLen = buffs.size();
- long curIndex = 0;
- for (index = 0; index < buffLen; index++) {
- final byte[] buff = buffs.get(index);
- if (curIndex + buff.length < offset) {
- curIndex += buff.length;
- continue;
- }
+ if (sliceBytes > 0) {
+ // simply some math calcs to determine the offset
+ index = (int) (offset / sliceBytes);
+ position = (int) (offset - (index * sliceBytes));
+ // position = (int) (offset % sliceBytes);
+ } else {
+ // loop the buffer to determine the offset
+ long curIndex = 0;
+ for (index = 0; index < buffLen; index++) {
+ final byte[] buff = buffs.get(index);
+ if (curIndex + buff.length < offset) {
+ curIndex += buff.length;
+ continue;
+ }
- // matched and calc the position
- position = (int) (offset - curIndex);
- break;
+ // matched and calc the position
+ position = (int) (offset - curIndex);
+ break;
+ }
}
return new Position(index, position);
diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java
index ed11e90..1786bfb 100644
--- a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java
+++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java
@@ -25,11 +25,15 @@ public class Searcher {
public static final int VectorIndexCols = 256;
public static final int VectorIndexSize = 8;
+ // maximum slice bytes for dynamic buffer array.
// Linux max write / read bytes.
// Check https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA
// to get to know why we default to this value.
public static final int MAX_WRITE_BYTES = 0x7ffff000;
+ // default slice bytes (50 MiB) for fixed buffer array.
+ public static final int DEFAULT_SLICE_BYTES = 50 * 1024 * 1024;
+
// ip version
private final Version version;
@@ -259,14 +263,14 @@ public class Searcher {
// -- load xdb buffer with random access file handle
public static LongByteArray loadContent(RandomAccessFile handle) throws IOException {
- return loadContent(handle, MAX_WRITE_BYTES);
+ return loadContent(handle, DEFAULT_SLICE_BYTES);
}
public static LongByteArray loadContent(RandomAccessFile handle, final int sliceBytes) throws IOException {
handle.seek(0);
// check the length and do the buff load
long toRead = handle.length();
- final LongByteArray byteArray = new LongByteArray();
+ final LongByteArray byteArray = new LongByteArray(sliceBytes);
while (toRead > 0) {
final byte[] buff = new byte[(int) Math.min(toRead, sliceBytes)];
final int rLen = handle.read(buff);
@@ -284,7 +288,7 @@ public class Searcher {
// -- load xdb buffer with xdb file object
public static LongByteArray loadContentFromFile(File xdbFile) throws IOException {
- return loadContentFromFile(xdbFile, MAX_WRITE_BYTES);
+ return loadContentFromFile(xdbFile, DEFAULT_SLICE_BYTES);
}
public static LongByteArray loadContentFromFile(File xdbFile, final int sliceBytes) throws IOException {
@@ -297,7 +301,7 @@ public class Searcher {
// -- load xdb buffer with xdb file path
public static LongByteArray loadContentFromFile(String xdbPath) throws IOException {
- return loadContentFromFile(xdbPath, MAX_WRITE_BYTES);
+ return loadContentFromFile(xdbPath, DEFAULT_SLICE_BYTES);
}
public static LongByteArray loadContentFromFile(String xdbPath, final int sliceBytes) throws IOException {
@@ -307,11 +311,11 @@ public class Searcher {
// load xdb buffer from input stream
public static LongByteArray loadContentFromInputStream(InputStream is) throws IOException {
- return loadContentFromInputStream(is, MAX_WRITE_BYTES);
+ return loadContentFromInputStream(is, DEFAULT_SLICE_BYTES);
}
public static LongByteArray loadContentFromInputStream(InputStream is, final int sliceBytes) throws IOException {
- final LongByteArray byteArray = new LongByteArray();
+ final LongByteArray byteArray = new LongByteArray(sliceBytes);
while (true) {
boolean done = false;
diff --git a/binding/java/src/test/java/org/lionsoul/ip2region/xdb/BufferTest.java b/binding/java/src/test/java/org/lionsoul/ip2region/xdb/BufferTest.java
index 59fa6b4..541ddd1 100644
--- a/binding/java/src/test/java/org/lionsoul/ip2region/xdb/BufferTest.java
+++ b/binding/java/src/test/java/org/lionsoul/ip2region/xdb/BufferTest.java
@@ -1,6 +1,8 @@
package org.lionsoul.ip2region.xdb;
+import java.io.File;
import java.io.FileInputStream;
+import java.io.IOException;
import java.security.CodeSource;
import org.junit.Test;
@@ -19,6 +21,8 @@ public class BufferTest {
}
}
+ // --- v4
+
@Test
public void testV4InputStreamBuffer() throws Exception {
final LongByteArray cBuffer = Searcher.loadContentFromInputStream(
@@ -27,6 +31,48 @@ public class BufferTest {
log.debugf("cBuffer->{length:%d, size:%d}", cBuffer.length(), cBuffer.size());
}
+ @Test
+ public void testV4FixedBuffer() throws Exception {
+ final LongByteArray cBuffer = Searcher.loadContentFromFile(
+ new File(getDataPath("ip2region_v4.xdb")), 2 * 1024 * 1024
+ );
+ final Header header = Searcher.loadHeaderFromBuffer(cBuffer);
+ log.debugf("cBuffer->{length:%d, size:%d}", cBuffer.length(), cBuffer.size());
+ log.debugf("Header->%s", header);
+ }
+
+ @Test
+ public void testV4BufferAssert() throws Exception {
+ final LongByteArray m2Bufer = Searcher.loadContentFromFile(
+ new File(getDataPath("ip2region_v4.xdb")), 2 * 1024 * 1024
+ );
+ final LongByteArray m5Bufer = Searcher.loadContentFromFile(
+ new File(getDataPath("ip2region_v4.xdb")), 5 * 1024 * 1024
+ );
+
+ final int[] offsets = new int[]{0, 10, 512, 1024, 39672, 1024 * 1024 * 2};
+ for (int idx : offsets) {
+ final long m2Val = m2Bufer.getUint32(idx);
+ final long m5Val = m5Bufer.getUint32(idx);
+ log.debugf("m2Buffer[%8d:4]: %10d, m5Buffer[%8d:4]: %10d, equals ? %s", idx, m2Val, idx, m5Val, m2Val == m5Val ? "true" : "false");
+ }
+ }
+
+ @Test
+ public void testV4BufferEOF() throws IOException {
+ final LongByteArray buffer = Searcher.loadContentFromFile(
+ new File(getDataPath("ip2region_v4.xdb")), 2 * 1024 * 1024
+ );
+
+ try {
+ buffer.append(new byte[1024]);
+ } catch (IOException e) {
+ log.debugf("failed to append: %s", e.getMessage());
+ }
+ }
+
+ // --- v6
+
@Test
public void testV6InputStreamBuffer() throws Exception {
final LongByteArray cBuffer = Searcher.loadContentFromInputStream(
@@ -35,4 +81,14 @@ public class BufferTest {
log.debugf("cBuffer->{length:%d, size:%d}", cBuffer.length(), cBuffer.size());
}
+ @Test
+ public void testV6FixedBuffer() throws Exception {
+ final LongByteArray cBuffer = Searcher.loadContentFromFile(
+ new File(getDataPath("ip2region_v6.xdb")), 5 * 1024 * 1024
+ );
+ final Header header = Searcher.loadHeaderFromBuffer(cBuffer);
+ log.debugf("cBuffer->{length:%d, size:%d}", cBuffer.length(), cBuffer.size());
+ log.debugf("Header->%s", header);
+ }
+
}