add in-memory search supports when the xdb file size exceeds 2^31 - 1

binding/java/pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>org.lionsoul</groupId>
     <artifactId>ip2region</artifactId>
-    <version>2.8.0</version>
+    <version>2.8.1</version>
     <packaging>jar</packaging>
 
     <name>ip2region</name>

binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java

@@ -6,6 +6,7 @@
 
 package org.lionsoul.ip2region;
 
+import org.lionsoul.ip2region.xdb.LongByteArray;
 import org.lionsoul.ip2region.xdb.Searcher;
 
 import java.io.*;
@@ -30,7 +31,7 @@ public class SearchTest {
             byte[] vIndex = Searcher.loadVectorIndexFromFile(dbPath);
             return Searcher.newWithVectorIndex(dbPath, vIndex);
         } else if ("content".equals(cachePolicy)) {
-            byte[] cBuff = Searcher.loadContentFromFile(dbPath);
+            LongByteArray cBuff = Searcher.loadContentFromFile(dbPath);
             return Searcher.newWithBuffer(cBuff);
         } else {
             throw new IOException("invalid cache policy `" + cachePolicy + "`, options: file/vectorIndex/content");

binding/java/src/main/java/org/lionsoul/ip2region/UtilTest.java

@@ -6,8 +6,11 @@
 
 package org.lionsoul.ip2region;
 
+import org.lionsoul.ip2region.xdb.LongByteArray;
 import org.lionsoul.ip2region.xdb.Searcher;
 
+import java.util.Arrays;
+
 public class UtilTest {
 
     public static void testIP2Long() {
@@ -34,9 +37,49 @@ public class UtilTest {
         System.out.printf("passed: ip=%s, ipAddr=%d, ip2=%s\n", ip, ipAddr, ip2);
     }
 
+    public static void testLongByteArray() {
+        final LongByteArray byteArray = new LongByteArray();
+        byteArray.append(new byte[]{0,0,0,0,0});
+        byteArray.append(new byte[]{1,1,1,1,1});
+        int counter = 2;
+        for (int i = 0; i < 100; i++) {
+            final byte[] buff = new byte[10];
+            Arrays.fill(buff, (byte) counter);
+            byteArray.append(buff);
+            counter++;
+        }
+
+        System.out.printf("1, byteArray.length: %d\n", byteArray.length());
+        System.out.println("2, length copy test...");
+        int[] length = new int[]{5, 10, 15, 20, 21, 22, 23, 25, 28, 29, 30, 40, 42, 44, 50, 60};
+        for (int j : length) {
+            final byte[] destBuff = new byte[j];
+            byteArray.copy(0, destBuff, 0, destBuff.length);
+            System.out.printf("copy(0,%d): \n", destBuff.length);
+            for (byte b : destBuff) {
+                System.out.print(b + " ");
+            }
+            System.out.println();
+        }
+
+        System.out.println("3, offset copy test...");
+        int[] offset = new int[]{0, 5, 10, 15, 20, 21, 22, 23, 25, 28, 29, 30, 40, 42, 44, 50, 60};
+        for (int j : offset) {
+            final byte[] destBuff = new byte[11];
+            byteArray.copy(j, destBuff, 0, destBuff.length);
+            System.out.printf("copy(%d,%d): \n", j, destBuff.length);
+            for (byte b : destBuff) {
+                System.out.print(b + " ");
+            }
+            System.out.println();
+        }
+    }
+
     public static void main(String[] args) {
         System.out.print("testing IP2Long ... \n");
         testIP2Long();
+        System.out.print("testing LongByteArray ... \n");
+        testLongByteArray();
     }
 
 }

binding/java/src/main/java/org/lionsoul/ip2region/xdb/LongByteArray.java

@@ -0,0 +1,140 @@
+// Copyright 2022 The Ip2Region Authors. All rights reserved.
+// Use of this source code is governed by a Apache2.0-style
+// license that can be found in the LICENSE file.
+
+package org.lionsoul.ip2region.xdb;
+
+// xdb byte buffer which used to instead of the byte array
+// when the size of the xdb file is greater than 2^32 << 2;
+// xdb file v4 is designed to be a maximum of 2^32 bytes in size.
+// @Author Leon <chenxin619315@gmail.com>
+// @Date 2025/08/22
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class LongByteArray {
+    // byte buffer list
+    private final List<byte[]> buffs = new ArrayList<byte[]>();
+    private long length;
+
+    public LongByteArray() {
+        this.length = 0;
+    }
+
+    public LongByteArray(byte[] buff) {
+        buffs.add(buff);
+        length = buff.length;
+    }
+
+    // append new buffer
+    public void append(final byte[] buffer) {
+        buffs.add(buffer);
+        length += buffer.length;
+    }
+
+    public long length() {
+        return length;
+    }
+
+    // internal method to determine the position of the specified offset
+    private Position determinate(final long offset) {
+        int index = 0, position = 0, buffLen = buffs.size();
+        long curIndex = 0;
+        for (index = 0; index < buffLen; index++) {
+            final byte[] buff = buffs.get(index);
+            if (curIndex + buff.length < offset) {
+                curIndex += buff.length;
+                continue;
+            }
+
+            // matched and calc the position
+            position = (int) (offset - curIndex);
+            break;
+        }
+
+        return new Position(index, position);
+    }
+
+    // Copy from the current buffers to a specified buffer
+    // from the specified offset with a specified length
+    public byte[] copy(final long srcPos, final byte[] dest, final int destPos, final int length) {
+        if (srcPos >= this.length) {
+            throw new IndexOutOfBoundsException("srcPos exceed the maximum array length `" + this.length + "`");
+        }
+
+        if (destPos + length > dest.length) {
+            throw new IndexOutOfBoundsException("destPost+length exceed the maximum dest buffer length `" + dest.length + "`");
+        }
+
+        final Position pos = determinate(srcPos);
+
+        // copy from the current buffer
+        final byte[] hBuff = buffs.get(pos.index++);
+        final int copyLen = Math.min(hBuff.length - pos.offset, length);
+        System.arraycopy(hBuff, pos.offset, dest, destPos, copyLen);
+
+        // check and copy from the rest buffer?
+        int sPos = destPos + copyLen;
+        int left = length - copyLen;
+        while (left > 0) {
+            final byte[] tBuff = buffs.get(pos.index++);
+            final int buffLen = tBuff.length;
+            if (left <= buffLen) {
+                System.arraycopy(tBuff, 0, dest, sPos, left);
+                break;
+            }
+
+            System.arraycopy(tBuff, 0, dest, sPos, buffLen);
+            sPos += buffLen;
+            left -= buffLen;
+        }
+
+        return dest;
+    }
+
+    // get a byte-buffer from the specified index with a specified length.
+    // this method will allocate a new byte buffer with length = $length.
+    public byte[] slice(long offset, int length) {
+        if (offset + length > this.length) {
+            throw new IndexOutOfBoundsException("offset+length exceed the maximum array length `" + this.length + "`");
+        }
+
+        final byte[] buffer = new  byte[length];
+        return copy(offset, buffer, 0, length);
+    }
+
+    // get a 4-bytes long integer from the specified index
+    public long getIntLong(long offset) {
+        final byte[] b = new byte[4];
+        copy(offset, b, 0, 4);
+        return (
+            ((b[0] & 0x000000FFL)) |
+            ((b[1] <<  8) & 0x0000FF00L) |
+            ((b[2] << 16) & 0x00FF0000L) |
+            ((b[3] << 24) & 0xFF000000L)
+        );
+    }
+
+    public int getInt(long offset) {
+        final byte[] b = new byte[4];
+        copy(offset, b, 0, 4);
+        return (
+            ((b[0] & 0x000000FF)) |
+            ((b[1] <<  8) & 0x0000FF00) |
+            ((b[2] << 16) & 0x00FF0000) |
+            ((b[3] << 24) & 0xFF000000)
+        );
+    }
+
+    // position entry class
+    public static class Position {
+        public int index;
+        public int offset;
+        public Position(int index, int offset) {
+            this.index = index;
+            this.offset = offset;
+        }
+    }
+
+}

binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java

@@ -20,7 +20,10 @@ public class Searcher {
     public static final int VectorIndexSize  = 8;
     public static final int SegmentIndexSize = 14;
 
-    // random access file handle for file based search
+    // Linux max write / read bytes
+    public static final int MAX_WRITE_BYTES = 0x7ffff000;
+
+    // random access file handle for file-based search
     private final RandomAccessFile handle;
 
     private int ioCount = 0;
@@ -30,8 +33,10 @@ public class Searcher {
     // the minimal memory allocation.
     private final byte[] vectorIndex;
 
-    // xdb content buffer, used for in-memory search
+    // xdb content buffer, used for in-memory search.
-    private final byte[] contentBuff;
+    // @Note: use the LongByteArray instead since 2025/08/22
+    // private final byte[] contentBuff;
+    private final LongByteArray contentBuff;
 
     // --- static method to create searchers
 
@@ -43,13 +48,13 @@ public class Searcher {
         return new Searcher(dbPath, vectorIndex, null);
     }
 
-    public static Searcher newWithBuffer(byte[] cBuff) throws IOException {
+    public static Searcher newWithBuffer(LongByteArray cBuff) throws IOException {
         return new Searcher(null, null, cBuff);
     }
 
     // --- End of creator
 
-    public Searcher(String dbFile, byte[] vectorIndex, byte[] cBuff) throws IOException {
+    public Searcher(String dbFile, byte[] vectorIndex, LongByteArray cBuff) throws IOException {
         if (cBuff != null) {
             this.handle = null;
             this.vectorIndex = null;
@@ -90,8 +95,8 @@ public class Searcher {
             sPtr = getIntLong(vectorIndex, idx);
             ePtr = getIntLong(vectorIndex, idx + 4);
         } else if (contentBuff != null) {
-            sPtr = getIntLong(contentBuff, HeaderInfoLength + idx);
+            sPtr = contentBuff.getIntLong(HeaderInfoLength + idx);
-            ePtr = getIntLong(contentBuff, HeaderInfoLength + idx + 4);
+            ePtr = contentBuff.getIntLong(HeaderInfoLength + idx + 4);
         } else {
             final byte[] buff = new byte[VectorIndexSize];
             read(HeaderInfoLength + idx, buff);
@@ -141,15 +146,7 @@ public class Searcher {
     protected void read(long offset, byte[] buffer) throws IOException {
         // check the in-memory buffer first
         if (contentBuff != null) {
-            // @TODO: reduce data copying, directly decode the data ?
+            contentBuff.copy(offset, buffer, 0, buffer.length);
-            // @TODO: added by Leon at 2025/06/10, when offset is negative and the content byte is not going to work.
-            // we need a better solution for the content buffer which is greater than (2^31 - 1 << 2)
-            int int_idx = (int) offset;
-            if (int_idx < 0) {
-                throw new IOException("No content buffer policy for NOW since the xdb is too large, use file or vectorIndex instead");
-            }
-
-            System.arraycopy(contentBuff, int_idx, buffer, 0, buffer.length);
             return;
         }
 
@@ -199,20 +196,28 @@ public class Searcher {
         return vIndex;
     }
 
-    public static byte[] loadContent(RandomAccessFile handle) throws IOException {
+    public static LongByteArray loadContent(RandomAccessFile handle) throws IOException {
         handle.seek(0);
-        final byte[] buff =  new byte[(int) handle.length()];
+        // check the length and do the buff load
-        int rLen = handle.read(buff);
+        long toRead = handle.length();
+        final LongByteArray byteArray = new LongByteArray();
+        while (toRead > 0) {
+            final byte[] buff = new byte[(int) Math.min(toRead, MAX_WRITE_BYTES)];
+            final int rLen = handle.read(buff);
             if (rLen != buff.length) {
-            throw new IOException("incomplete read: read bytes should be " + buff.length);
+                throw new IOException("incomplete read: read bytes should be " + buff.length + ", got `" + rLen + "`");
             }
 
-        return buff;
+            byteArray.append(buff);
+            toRead -= rLen;
         }
 
-    public static byte[] loadContentFromFile(String dbPath) throws IOException {
+        return byteArray;
+    }
+
+    public static LongByteArray loadContentFromFile(String dbPath) throws IOException {
         final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
-        final byte[] content = loadContent(handle);
+        final LongByteArray content = loadContent(handle);
         handle.close();
         return content;
     }

binding/php/batch_test.php

@@ -112,6 +112,7 @@ if ($handle === false) {
 }
 
 $count = 0;
+$qx_count = 0;
 while (!feof($handle)) {
     $line = trim(fgets($handle, 1024));
     if (strlen($line) < 1) {
@@ -126,8 +127,13 @@ while (!feof($handle)) {
 
     $count++;
     $region = $searcher->search($ip);
-    echo $line, ", ", $region, "\n";
+    $ss = explode('|', $region);
+    if (strlen($ss[3]) > 1) {
+        $qx_count++;
+    }
+    echo $line, ",", str_replace('|', ',', $region), "\n";
 }
 
 fclose($handle);
+echo "qx_count: {$qx_count}";
 echo "Done, with {$count} IPs\n";