diff --git a/binding/java/pom.xml b/binding/java/pom.xml
index 09481a1..d531daf 100644
--- a/binding/java/pom.xml
+++ b/binding/java/pom.xml
@@ -4,7 +4,7 @@
org.lionsoul
ip2region
- 2.8.0
+ 2.8.1
jar
ip2region
diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java b/binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java
index 23d48e1..bc6b53f 100644
--- a/binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java
+++ b/binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java
@@ -6,6 +6,7 @@
package org.lionsoul.ip2region;
+import org.lionsoul.ip2region.xdb.LongByteArray;
import org.lionsoul.ip2region.xdb.Searcher;
import java.io.*;
@@ -30,7 +31,7 @@ public class SearchTest {
byte[] vIndex = Searcher.loadVectorIndexFromFile(dbPath);
return Searcher.newWithVectorIndex(dbPath, vIndex);
} else if ("content".equals(cachePolicy)) {
- byte[] cBuff = Searcher.loadContentFromFile(dbPath);
+ LongByteArray cBuff = Searcher.loadContentFromFile(dbPath);
return Searcher.newWithBuffer(cBuff);
} else {
throw new IOException("invalid cache policy `" + cachePolicy + "`, options: file/vectorIndex/content");
diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/UtilTest.java b/binding/java/src/main/java/org/lionsoul/ip2region/UtilTest.java
index aba7779..db5cbbb 100644
--- a/binding/java/src/main/java/org/lionsoul/ip2region/UtilTest.java
+++ b/binding/java/src/main/java/org/lionsoul/ip2region/UtilTest.java
@@ -6,8 +6,11 @@
package org.lionsoul.ip2region;
+import org.lionsoul.ip2region.xdb.LongByteArray;
import org.lionsoul.ip2region.xdb.Searcher;
+import java.util.Arrays;
+
public class UtilTest {
public static void testIP2Long() {
@@ -34,9 +37,49 @@ public class UtilTest {
System.out.printf("passed: ip=%s, ipAddr=%d, ip2=%s\n", ip, ipAddr, ip2);
}
+ public static void testLongByteArray() {
+ final LongByteArray byteArray = new LongByteArray();
+ byteArray.append(new byte[]{0,0,0,0,0});
+ byteArray.append(new byte[]{1,1,1,1,1});
+ int counter = 2;
+ for (int i = 0; i < 100; i++) {
+ final byte[] buff = new byte[10];
+ Arrays.fill(buff, (byte) counter);
+ byteArray.append(buff);
+ counter++;
+ }
+
+ System.out.printf("1, byteArray.length: %d\n", byteArray.length());
+ System.out.println("2, length copy test...");
+ int[] length = new int[]{5, 10, 15, 20, 21, 22, 23, 25, 28, 29, 30, 40, 42, 44, 50, 60};
+ for (int j : length) {
+ final byte[] destBuff = new byte[j];
+ byteArray.copy(0, destBuff, 0, destBuff.length);
+ System.out.printf("copy(0,%d): \n", destBuff.length);
+ for (byte b : destBuff) {
+ System.out.print(b + " ");
+ }
+ System.out.println();
+ }
+
+ System.out.println("3, offset copy test...");
+ int[] offset = new int[]{0, 5, 10, 15, 20, 21, 22, 23, 25, 28, 29, 30, 40, 42, 44, 50, 60};
+ for (int j : offset) {
+ final byte[] destBuff = new byte[11];
+ byteArray.copy(j, destBuff, 0, destBuff.length);
+ System.out.printf("copy(%d,%d): \n", j, destBuff.length);
+ for (byte b : destBuff) {
+ System.out.print(b + " ");
+ }
+ System.out.println();
+ }
+ }
+
public static void main(String[] args) {
System.out.print("testing IP2Long ... \n");
testIP2Long();
+ System.out.print("testing LongByteArray ... \n");
+ testLongByteArray();
}
}
diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/LongByteArray.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/LongByteArray.java
new file mode 100644
index 0000000..aa85b6b
--- /dev/null
+++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/LongByteArray.java
@@ -0,0 +1,140 @@
+// Copyright 2022 The Ip2Region Authors. All rights reserved.
+// Use of this source code is governed by a Apache2.0-style
+// license that can be found in the LICENSE file.
+
+package org.lionsoul.ip2region.xdb;
+
+// xdb byte buffer which used to instead of the byte array
+// when the size of the xdb file is greater than 2^32 << 2;
+// xdb file v4 is designed to be a maximum of 2^32 bytes in size.
+// @Author Leon
+// @Date 2025/08/22
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class LongByteArray {
+ // byte buffer list
+ private final List buffs = new ArrayList();
+ private long length;
+
+ public LongByteArray() {
+ this.length = 0;
+ }
+
+ public LongByteArray(byte[] buff) {
+ buffs.add(buff);
+ length = buff.length;
+ }
+
+ // append new buffer
+ public void append(final byte[] buffer) {
+ buffs.add(buffer);
+ length += buffer.length;
+ }
+
+ public long length() {
+ return length;
+ }
+
+ // internal method to determine the position of the specified offset
+ private Position determinate(final long offset) {
+ int index = 0, position = 0, buffLen = buffs.size();
+ long curIndex = 0;
+ for (index = 0; index < buffLen; index++) {
+ final byte[] buff = buffs.get(index);
+ if (curIndex + buff.length < offset) {
+ curIndex += buff.length;
+ continue;
+ }
+
+ // matched and calc the position
+ position = (int) (offset - curIndex);
+ break;
+ }
+
+ return new Position(index, position);
+ }
+
+ // Copy from the current buffers to a specified buffer
+ // from the specified offset with a specified length
+ public byte[] copy(final long srcPos, final byte[] dest, final int destPos, final int length) {
+ if (srcPos >= this.length) {
+ throw new IndexOutOfBoundsException("srcPos exceed the maximum array length `" + this.length + "`");
+ }
+
+ if (destPos + length > dest.length) {
+ throw new IndexOutOfBoundsException("destPost+length exceed the maximum dest buffer length `" + dest.length + "`");
+ }
+
+ final Position pos = determinate(srcPos);
+
+ // copy from the current buffer
+ final byte[] hBuff = buffs.get(pos.index++);
+ final int copyLen = Math.min(hBuff.length - pos.offset, length);
+ System.arraycopy(hBuff, pos.offset, dest, destPos, copyLen);
+
+ // check and copy from the rest buffer?
+ int sPos = destPos + copyLen;
+ int left = length - copyLen;
+ while (left > 0) {
+ final byte[] tBuff = buffs.get(pos.index++);
+ final int buffLen = tBuff.length;
+ if (left <= buffLen) {
+ System.arraycopy(tBuff, 0, dest, sPos, left);
+ break;
+ }
+
+ System.arraycopy(tBuff, 0, dest, sPos, buffLen);
+ sPos += buffLen;
+ left -= buffLen;
+ }
+
+ return dest;
+ }
+
+ // get a byte-buffer from the specified index with a specified length.
+ // this method will allocate a new byte buffer with length = $length.
+ public byte[] slice(long offset, int length) {
+ if (offset + length > this.length) {
+ throw new IndexOutOfBoundsException("offset+length exceed the maximum array length `" + this.length + "`");
+ }
+
+ final byte[] buffer = new byte[length];
+ return copy(offset, buffer, 0, length);
+ }
+
+ // get a 4-bytes long integer from the specified index
+ public long getIntLong(long offset) {
+ final byte[] b = new byte[4];
+ copy(offset, b, 0, 4);
+ return (
+ ((b[0] & 0x000000FFL)) |
+ ((b[1] << 8) & 0x0000FF00L) |
+ ((b[2] << 16) & 0x00FF0000L) |
+ ((b[3] << 24) & 0xFF000000L)
+ );
+ }
+
+ public int getInt(long offset) {
+ final byte[] b = new byte[4];
+ copy(offset, b, 0, 4);
+ return (
+ ((b[0] & 0x000000FF)) |
+ ((b[1] << 8) & 0x0000FF00) |
+ ((b[2] << 16) & 0x00FF0000) |
+ ((b[3] << 24) & 0xFF000000)
+ );
+ }
+
+ // position entry class
+ public static class Position {
+ public int index;
+ public int offset;
+ public Position(int index, int offset) {
+ this.index = index;
+ this.offset = offset;
+ }
+ }
+
+}
diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java
index f06da2c..52fe054 100644
--- a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java
+++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java
@@ -20,7 +20,10 @@ public class Searcher {
public static final int VectorIndexSize = 8;
public static final int SegmentIndexSize = 14;
- // random access file handle for file based search
+ // Linux max write / read bytes
+ public static final int MAX_WRITE_BYTES = 0x7ffff000;
+
+ // random access file handle for file-based search
private final RandomAccessFile handle;
private int ioCount = 0;
@@ -30,8 +33,10 @@ public class Searcher {
// the minimal memory allocation.
private final byte[] vectorIndex;
- // xdb content buffer, used for in-memory search
- private final byte[] contentBuff;
+ // xdb content buffer, used for in-memory search.
+ // @Note: use the LongByteArray instead since 2025/08/22
+ // private final byte[] contentBuff;
+ private final LongByteArray contentBuff;
// --- static method to create searchers
@@ -43,13 +48,13 @@ public class Searcher {
return new Searcher(dbPath, vectorIndex, null);
}
- public static Searcher newWithBuffer(byte[] cBuff) throws IOException {
+ public static Searcher newWithBuffer(LongByteArray cBuff) throws IOException {
return new Searcher(null, null, cBuff);
}
// --- End of creator
- public Searcher(String dbFile, byte[] vectorIndex, byte[] cBuff) throws IOException {
+ public Searcher(String dbFile, byte[] vectorIndex, LongByteArray cBuff) throws IOException {
if (cBuff != null) {
this.handle = null;
this.vectorIndex = null;
@@ -90,8 +95,8 @@ public class Searcher {
sPtr = getIntLong(vectorIndex, idx);
ePtr = getIntLong(vectorIndex, idx + 4);
} else if (contentBuff != null) {
- sPtr = getIntLong(contentBuff, HeaderInfoLength + idx);
- ePtr = getIntLong(contentBuff, HeaderInfoLength + idx + 4);
+ sPtr = contentBuff.getIntLong(HeaderInfoLength + idx);
+ ePtr = contentBuff.getIntLong(HeaderInfoLength + idx + 4);
} else {
final byte[] buff = new byte[VectorIndexSize];
read(HeaderInfoLength + idx, buff);
@@ -141,15 +146,7 @@ public class Searcher {
protected void read(long offset, byte[] buffer) throws IOException {
// check the in-memory buffer first
if (contentBuff != null) {
- // @TODO: reduce data copying, directly decode the data ?
- // @TODO: added by Leon at 2025/06/10, when offset is negative and the content byte is not going to work.
- // we need a better solution for the content buffer which is greater than (2^31 - 1 << 2)
- int int_idx = (int) offset;
- if (int_idx < 0) {
- throw new IOException("No content buffer policy for NOW since the xdb is too large, use file or vectorIndex instead");
- }
-
- System.arraycopy(contentBuff, int_idx, buffer, 0, buffer.length);
+ contentBuff.copy(offset, buffer, 0, buffer.length);
return;
}
@@ -199,20 +196,28 @@ public class Searcher {
return vIndex;
}
- public static byte[] loadContent(RandomAccessFile handle) throws IOException {
+ public static LongByteArray loadContent(RandomAccessFile handle) throws IOException {
handle.seek(0);
- final byte[] buff = new byte[(int) handle.length()];
- int rLen = handle.read(buff);
- if (rLen != buff.length) {
- throw new IOException("incomplete read: read bytes should be " + buff.length);
+ // check the length and do the buff load
+ long toRead = handle.length();
+ final LongByteArray byteArray = new LongByteArray();
+ while (toRead > 0) {
+ final byte[] buff = new byte[(int) Math.min(toRead, MAX_WRITE_BYTES)];
+ final int rLen = handle.read(buff);
+ if (rLen != buff.length) {
+ throw new IOException("incomplete read: read bytes should be " + buff.length + ", got `" + rLen + "`");
+ }
+
+ byteArray.append(buff);
+ toRead -= rLen;
}
- return buff;
+ return byteArray;
}
- public static byte[] loadContentFromFile(String dbPath) throws IOException {
+ public static LongByteArray loadContentFromFile(String dbPath) throws IOException {
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
- final byte[] content = loadContent(handle);
+ final LongByteArray content = loadContent(handle);
handle.close();
return content;
}
diff --git a/binding/php/batch_test.php b/binding/php/batch_test.php
index ef144ea..14ecd37 100644
--- a/binding/php/batch_test.php
+++ b/binding/php/batch_test.php
@@ -112,6 +112,7 @@ if ($handle === false) {
}
$count = 0;
+$qx_count = 0;
while (!feof($handle)) {
$line = trim(fgets($handle, 1024));
if (strlen($line) < 1) {
@@ -126,8 +127,13 @@ while (!feof($handle)) {
$count++;
$region = $searcher->search($ip);
- echo $line, ", ", $region, "\n";
+ $ss = explode('|', $region);
+ if (strlen($ss[3]) > 1) {
+ $qx_count++;
+ }
+ echo $line, ",", str_replace('|', ',', $region), "\n";
}
fclose($handle);
+echo "qx_count: {$qx_count}";
echo "Done, with {$count} IPs\n";