Merge pull request #411 from lionsoul2014/fr_java_fix_slice_bytes
Fr java fixed slice bytes
This commit is contained in:
Leon / 狮子的魂
GitHub
commit
351be412e6
|
|
@ -7,7 +7,7 @@
|
|||
<dependency>
|
||||
<groupId>org.lionsoul</groupId>
|
||||
<artifactId>ip2region</artifactId>
|
||||
<version>3.3.2</version>
|
||||
<version>3.3.3</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
|
|
@ -58,8 +58,7 @@ ip2Region.close();
|
|||
final Ip2Region ip2region = Ip2Region.create(v4Config, v6Config, true);
|
||||
```
|
||||
4. 如果配置设置的缓存策略为 `Config.BufferCache` 即 `全内存缓存` 则默认会使用单实例的内存查询器，该实现天生并发安全，此时通过 `setSearchers` 指定的查询器数量无效。
|
||||
5. 如果使用的是全内存缓存查询且在加载 xdb 二进制内容的时候提示 `OOM`，请参考 [sliceBytes设置](#slicebytes) 然后通过 `ConfigBuilder.setCacheSliceBytes(int)` 设置一个合适的值来避免 OOM。
|
||||
6. 如果 `ip2region` 查询器在提供服务期间，调用 close 默认会最大等待 10 秒钟来等待尽量多的查询器归还。
|
||||
5. 如果 `ip2region` 查询器在提供服务期间，调用 close 默认会最大等待 10 秒钟来等待尽量多的查询器归还。
|
||||
|
||||
|
||||
### 关于查询 API
|
||||
|
|
@ -245,10 +244,10 @@ public class SearcherTest {
|
|||
|
||||
### sliceBytes
|
||||
|
||||
sliceBytes 表示 xdb 全内存缓存时 `LongByteArray` 类内部维护的 `List<byte[]> buffs` 集合的分片内存的大小，这个值的最大值也是默认值为 `Searcher.MAX_WRITE_BYTES`，取值的核心是为了减少 `buffs` 的长度， 最小值为 1，buffs 长度越小越好，意味着查询过程中的寻址操作的 buffs 遍历操作越少，该值的设置原则如下：
|
||||
1. 默认为 `Searcher.MAX_WRITE_BYTES`，也就是 `0x7ffff000`，关于该取值的来源可以参考作者博客文章：[https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA](https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA)。
|
||||
2. 如果 xdb 文件的字节数小于 `Searcher.MAX_WRITE_BYTES`，则 sliceBytes 设置为该 xdb 文件的字节数即可，如果大于 `Searcher.MAX_WRITE_BYTES` 则使用默认值即可。Searcher 的 `loadContent` 或者 `loadContentFromFile` 方法默认都是按照这个原则来自动设置 sliceBytes 的值，唯独 `loadContentFromInputStream` 系列方法因为不方便获取流的大小使用的是默认最大值，因此可以按照上述原则通过调用 `loadContentFromInputStream(InputStream, int)` 手动设置合理的值，或者设置 JVM 的内存限制避免运行时的 OOM 错误。
|
||||
3. 如果 sliceBytes 设置的值小于甚至远远小于 xdb 文件的字节数则会增加查询过程中的寻址遍历操作从而减慢查询，其他无任何影响，随着 IPv6 的普及后期的 xdb 文件大小可能几个G甚至10G+，所以默认 sliceBytes 取的最大值也是为了默认总是能取得最佳的运行效率。
|
||||
sliceBytes 表示 xdb 全内存缓存时 `LongByteArray` 类内部维护的 `List<byte[]> buffs` 集合的分片内存的大小，默认值为 `Searcher.DEFAULT_SLICE_BYTES` = `50MiB`，这个值的最大允许值为 `Searcher.MAX_WRITE_BYTES` = `0x7ffff000`，关于该取值的来源可以参考作者博客文章：[https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA](https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA)。
|
||||
1. 从 `3.3.3` 版本开始 `LongByteArray` 实现了固定分片尺寸支持，可以通过简单的计算快速的完成 `offset` 定位的从而实现 `slice` 或者 `copy` 操作。
|
||||
2. 从计算速度来说 sliceBytes 越大 buffs 的长度越小，计算耗时越小，不过有了固定 sliceBytes 实现这个差距完全可以忽略，所以建议保持默认值为 `50MiB` 即可，也不会出现之前弹性分片尺寸可能导致的 OOM 问题。
|
||||
|
||||
|
||||
|
||||
# 编译测试程序
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
<groupId>org.lionsoul</groupId>
|
||||
<artifactId>ip2region</artifactId>
|
||||
<version>3.3.2</version>
|
||||
<version>3.3.3</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>ip2region</name>
|
||||
|
|
|
|||
|
|
@ -239,6 +239,9 @@ public class SearcherTest {
|
|||
return;
|
||||
}
|
||||
|
||||
// mark the start time
|
||||
long sTime = System.nanoTime();
|
||||
|
||||
byte[] sip;
|
||||
try {
|
||||
sip = Util.parseIP(ps[0]);
|
||||
|
|
@ -264,9 +267,7 @@ public class SearcherTest {
|
|||
}
|
||||
|
||||
for (final byte[] ip : new byte[][]{sip, eip}) {
|
||||
long sTime = System.nanoTime();
|
||||
String region = searcher.search(ip);
|
||||
costs += System.nanoTime() - sTime;
|
||||
|
||||
// check the region info
|
||||
if (!ps[2].equals(region)) {
|
||||
|
|
@ -277,6 +278,8 @@ public class SearcherTest {
|
|||
|
||||
count++;
|
||||
}
|
||||
|
||||
costs += System.nanoTime() - sTime;
|
||||
}
|
||||
|
||||
reader.close();
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@
|
|||
|
||||
package org.lionsoul.ip2region.xdb;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
// xdb byte buffer which used to instead of the byte array
|
||||
// when the size of the xdb file is greater than 2^32 << 2;
|
||||
// xdb file v4 is designed to be a maximum of 2^32 bytes in size.
|
||||
|
|
@ -14,21 +16,44 @@ import java.util.ArrayList;
|
|||
import java.util.List;
|
||||
|
||||
public class LongByteArray {
|
||||
|
||||
// slice bytes
|
||||
// if it is greater than the 0 we will use the fixed slice bytes
|
||||
// or we use the dynamic slice bytes.
|
||||
private final int sliceBytes;
|
||||
|
||||
// when EOF is true means we cannot call the #append anymore.
|
||||
// for fixed slice bytes only.
|
||||
private boolean _eof = false;
|
||||
|
||||
// byte buffer list
|
||||
private final List<byte[]> buffs = new ArrayList<byte[]>();
|
||||
private long length;
|
||||
|
||||
public LongByteArray() {
|
||||
this.length = 0;
|
||||
this.sliceBytes = -1;
|
||||
}
|
||||
|
||||
public LongByteArray(byte[] buff) {
|
||||
buffs.add(buff);
|
||||
length = buff.length;
|
||||
public LongByteArray(int sliceBytes) {
|
||||
assert sliceBytes != 0;
|
||||
assert sliceBytes <= Searcher.MAX_WRITE_BYTES;
|
||||
this.sliceBytes = sliceBytes;
|
||||
}
|
||||
|
||||
// append new buffer
|
||||
public void append(final byte[] buffer) {
|
||||
public void append(final byte[] buffer) throws IOException{
|
||||
// check and assert the slice bytes
|
||||
if (sliceBytes > 0) {
|
||||
if (_eof) {
|
||||
throw new IOException("buffer array closed (EOF=true)");
|
||||
} else if (buffer.length != sliceBytes) {
|
||||
// mark the buffer array as closed
|
||||
// since the last buffer block bytes is not equal to the expected #sliceBytes
|
||||
_eof = true;
|
||||
}
|
||||
}
|
||||
|
||||
buffs.add(buffer);
|
||||
length += buffer.length;
|
||||
}
|
||||
|
|
@ -44,17 +69,25 @@ public class LongByteArray {
|
|||
// internal method to determine the position of the specified offset
|
||||
private Position determinate(final long offset) {
|
||||
int index = 0, position = 0, buffLen = buffs.size();
|
||||
long curIndex = 0;
|
||||
for (index = 0; index < buffLen; index++) {
|
||||
final byte[] buff = buffs.get(index);
|
||||
if (curIndex + buff.length < offset) {
|
||||
curIndex += buff.length;
|
||||
continue;
|
||||
}
|
||||
if (sliceBytes > 0) {
|
||||
// simply some math calcs to determine the offset
|
||||
index = (int) (offset / sliceBytes);
|
||||
position = (int) (offset - (index * sliceBytes));
|
||||
// position = (int) (offset % sliceBytes);
|
||||
} else {
|
||||
// loop the buffer to determine the offset
|
||||
long curIndex = 0;
|
||||
for (index = 0; index < buffLen; index++) {
|
||||
final byte[] buff = buffs.get(index);
|
||||
if (curIndex + buff.length < offset) {
|
||||
curIndex += buff.length;
|
||||
continue;
|
||||
}
|
||||
|
||||
// matched and calc the position
|
||||
position = (int) (offset - curIndex);
|
||||
break;
|
||||
// matched and calc the position
|
||||
position = (int) (offset - curIndex);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return new Position(index, position);
|
||||
|
|
|
|||
|
|
@ -25,11 +25,15 @@ public class Searcher {
|
|||
public static final int VectorIndexCols = 256;
|
||||
public static final int VectorIndexSize = 8;
|
||||
|
||||
// maximum slice bytes for dynamic buffer array.
|
||||
// Linux max write / read bytes.
|
||||
// Check https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA
|
||||
// to get to know why we default to this value.
|
||||
public static final int MAX_WRITE_BYTES = 0x7ffff000;
|
||||
|
||||
// default slice bytes (50 MiB) for fixed buffer array.
|
||||
public static final int DEFAULT_SLICE_BYTES = 50 * 1024 * 1024;
|
||||
|
||||
// ip version
|
||||
private final Version version;
|
||||
|
||||
|
|
@ -259,14 +263,14 @@ public class Searcher {
|
|||
// -- load xdb buffer with random access file handle
|
||||
|
||||
public static LongByteArray loadContent(RandomAccessFile handle) throws IOException {
|
||||
return loadContent(handle, MAX_WRITE_BYTES);
|
||||
return loadContent(handle, DEFAULT_SLICE_BYTES);
|
||||
}
|
||||
|
||||
public static LongByteArray loadContent(RandomAccessFile handle, final int sliceBytes) throws IOException {
|
||||
handle.seek(0);
|
||||
// check the length and do the buff load
|
||||
long toRead = handle.length();
|
||||
final LongByteArray byteArray = new LongByteArray();
|
||||
final LongByteArray byteArray = new LongByteArray(sliceBytes);
|
||||
while (toRead > 0) {
|
||||
final byte[] buff = new byte[(int) Math.min(toRead, sliceBytes)];
|
||||
final int rLen = handle.read(buff);
|
||||
|
|
@ -284,7 +288,7 @@ public class Searcher {
|
|||
// -- load xdb buffer with xdb file object
|
||||
|
||||
public static LongByteArray loadContentFromFile(File xdbFile) throws IOException {
|
||||
return loadContentFromFile(xdbFile, MAX_WRITE_BYTES);
|
||||
return loadContentFromFile(xdbFile, DEFAULT_SLICE_BYTES);
|
||||
}
|
||||
|
||||
public static LongByteArray loadContentFromFile(File xdbFile, final int sliceBytes) throws IOException {
|
||||
|
|
@ -297,7 +301,7 @@ public class Searcher {
|
|||
// -- load xdb buffer with xdb file path
|
||||
|
||||
public static LongByteArray loadContentFromFile(String xdbPath) throws IOException {
|
||||
return loadContentFromFile(xdbPath, MAX_WRITE_BYTES);
|
||||
return loadContentFromFile(xdbPath, DEFAULT_SLICE_BYTES);
|
||||
}
|
||||
|
||||
public static LongByteArray loadContentFromFile(String xdbPath, final int sliceBytes) throws IOException {
|
||||
|
|
@ -307,11 +311,11 @@ public class Searcher {
|
|||
// load xdb buffer from input stream
|
||||
|
||||
public static LongByteArray loadContentFromInputStream(InputStream is) throws IOException {
|
||||
return loadContentFromInputStream(is, MAX_WRITE_BYTES);
|
||||
return loadContentFromInputStream(is, DEFAULT_SLICE_BYTES);
|
||||
}
|
||||
|
||||
public static LongByteArray loadContentFromInputStream(InputStream is, final int sliceBytes) throws IOException {
|
||||
final LongByteArray byteArray = new LongByteArray();
|
||||
final LongByteArray byteArray = new LongByteArray(sliceBytes);
|
||||
while (true) {
|
||||
boolean done = false;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
package org.lionsoul.ip2region.xdb;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.security.CodeSource;
|
||||
|
||||
import org.junit.Test;
|
||||
|
|
@ -19,6 +21,8 @@ public class BufferTest {
|
|||
}
|
||||
}
|
||||
|
||||
// --- v4
|
||||
|
||||
@Test
|
||||
public void testV4InputStreamBuffer() throws Exception {
|
||||
final LongByteArray cBuffer = Searcher.loadContentFromInputStream(
|
||||
|
|
@ -27,6 +31,48 @@ public class BufferTest {
|
|||
log.debugf("cBuffer->{length:%d, size:%d}", cBuffer.length(), cBuffer.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testV4FixedBuffer() throws Exception {
|
||||
final LongByteArray cBuffer = Searcher.loadContentFromFile(
|
||||
new File(getDataPath("ip2region_v4.xdb")), 2 * 1024 * 1024
|
||||
);
|
||||
final Header header = Searcher.loadHeaderFromBuffer(cBuffer);
|
||||
log.debugf("cBuffer->{length:%d, size:%d}", cBuffer.length(), cBuffer.size());
|
||||
log.debugf("Header->%s", header);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testV4BufferAssert() throws Exception {
|
||||
final LongByteArray m2Bufer = Searcher.loadContentFromFile(
|
||||
new File(getDataPath("ip2region_v4.xdb")), 2 * 1024 * 1024
|
||||
);
|
||||
final LongByteArray m5Bufer = Searcher.loadContentFromFile(
|
||||
new File(getDataPath("ip2region_v4.xdb")), 5 * 1024 * 1024
|
||||
);
|
||||
|
||||
final int[] offsets = new int[]{0, 10, 512, 1024, 39672, 1024 * 1024 * 2};
|
||||
for (int idx : offsets) {
|
||||
final long m2Val = m2Bufer.getUint32(idx);
|
||||
final long m5Val = m5Bufer.getUint32(idx);
|
||||
log.debugf("m2Buffer[%8d:4]: %10d, m5Buffer[%8d:4]: %10d, equals ? %s", idx, m2Val, idx, m5Val, m2Val == m5Val ? "true" : "false");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testV4BufferEOF() throws IOException {
|
||||
final LongByteArray buffer = Searcher.loadContentFromFile(
|
||||
new File(getDataPath("ip2region_v4.xdb")), 2 * 1024 * 1024
|
||||
);
|
||||
|
||||
try {
|
||||
buffer.append(new byte[1024]);
|
||||
} catch (IOException e) {
|
||||
log.debugf("failed to append: %s", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// --- v6
|
||||
|
||||
@Test
|
||||
public void testV6InputStreamBuffer() throws Exception {
|
||||
final LongByteArray cBuffer = Searcher.loadContentFromInputStream(
|
||||
|
|
@ -35,4 +81,14 @@ public class BufferTest {
|
|||
log.debugf("cBuffer->{length:%d, size:%d}", cBuffer.length(), cBuffer.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testV6FixedBuffer() throws Exception {
|
||||
final LongByteArray cBuffer = Searcher.loadContentFromFile(
|
||||
new File(getDataPath("ip2region_v6.xdb")), 5 * 1024 * 1024
|
||||
);
|
||||
final Header header = Searcher.loadHeaderFromBuffer(cBuffer);
|
||||
log.debugf("cBuffer->{length:%d, size:%d}", cBuffer.length(), cBuffer.size());
|
||||
log.debugf("Header->%s", header);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue