xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: ref interface Searcher and impl

This commit is contained in:
liwei 2022-08-06 21:01:34 +08:00
parent 4dab11be98
commit 114beaa720
13 changed files with 634 additions and 448 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

172
binding/java/ReadMe.md
View File

@ -11,140 +11,57 @@
</dependency> </dependency>
``` ```
### 完全基于文件的查询 ### Example
```java ```java
package org.lionsoul.ip2region.example;
import org.lionsoul.ip2region.xdb.Searcher; import org.lionsoul.ip2region.xdb.Searcher;
import java.io.*;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
public class SearcherTest { /**
public static void main(String[] args) { * @see org.lionsoul.ip2region.xdb.Searcher
// 1、创建 searcher 对象 */
public class SearcherExample {
public static void main(String[] args) throws Exception {
String dbPath = "ip2region.xdb file path"; String dbPath = "ip2region.xdb file path";
Searcher searcher = null;
try { // 基于文件查询，单次 search 需要文件 IO，非线程安全，性能最差
searcher = Searcher.newWithFileOnly(dbPath); try (Searcher fileSearcher = Searcher.newWithFileOnly(dbPath)) {
} catch (IOException e) { searchExample(fileSearcher);
System.out.printf("failed to create searcher with `%s`: %s\n", dbPath, e);
return;
} }
// 2、查询 // 基于 VectorIndex 查询，单次 search 需要内存查询 + 文件 IO，非线程安全，性能次之
try (Searcher indexSearcher = Searcher.newWithVectorIndex(dbPath)) {
searchExample(indexSearcher);
}
// 推荐：基于 Buffer 查询，单次 search 全使用内存查询，线程安全，支持序列化，性能最佳
// 可重写 loadFile 方法扩展从其他文件系统(hdfs/s3/oss...)装载 buffer 数据
try (Searcher bufferSearcher = Searcher.newWithBuffer(dbPath)) {
searchExample(bufferSearcher);
}
// 备注：并发查询时，对于非线程安全实现推荐为每个线程创建一个独立的 searcher 对象单独使用。
}
private static void searchExample(Searcher searcher) {
String ip = "1.2.3.4";
try { try {
String ip = "1.2.3.4";
long sTime = System.nanoTime(); long sTime = System.nanoTime();
String region = searcher.search(ip); Searcher.Region region = searcher.searchRegion(ip);
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime);
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); System.out.printf("{region: %s, searcher: %s, ioCount: %d, took: %d μs}\n",
region, searcher.getClass().getSimpleName(), region.getIoCount(), cost);
System.out.println(region.toRegionMsg());
} catch (Exception e) { } catch (Exception e) {
System.out.printf("failed to search(%s): %s\n", ip, e); System.out.printf("failed to search(%s): %s\n", ip, e);
} }
// 3、关闭资源
searcher.close();
// 备注：并发使用，每个线程需要创建一个独立的 searcher 对象单独使用。
} }
} }
```
### 缓存 `VectorIndex` 索引
我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据，然后全局缓存，每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作，从而加速查询，减少 IO 压力。
```java
import org.lionsoul.ip2region.xdb.Searcher;
import java.io.*;
import java.util.concurrent.TimeUnit;
public class SearcherTest {
public static void main(String[] args) {
String dbPath = "ip2region.xdb file path";
// 1、从 dbPath 中预先加载 VectorIndex 缓存，并且把这个得到的数据作为全局变量，后续反复使用。
byte[] vIndex;
try {
vIndex = Searcher.loadVectorIndexFromFile(dbPath);
} catch (Exception e) {
System.out.printf("failed to load vector index from `%s`: %s\n", dbPath, e);
return;
}
// 2、使用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。
Searcher searcher;
try {
searcher = Searcher.newWithVectorIndex(dbPath, vIndex);
} catch (Exception e) {
System.out.printf("failed to create vectorIndex cached searcher with `%s`: %s\n", dbPath, e);
return;
}
// 3、查询
try {
String ip = "1.2.3.4";
long sTime = System.nanoTime();
String region = searcher.search(ip);
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
} catch (Exception e) {
System.out.printf("failed to search(%s): %s\n", ip, e);
}
// 4、关闭资源
searcher.close();
// 备注：每个线程需要单独创建一个独立的 Searcher 对象，但是都共享全局的制度 vIndex 缓存。
}
}
```
### 缓存整个 `xdb` 数据
我们也可以预先加载整个 ip2region.xdb 的数据到内存，然后基于这个数据创建查询对象来实现完全基于文件的查询，类似之前的 memory search。
```java
import org.lionsoul.ip2region.xdb.Searcher;
import java.io.*;
import java.util.concurrent.TimeUnit;
public class SearcherTest {
public static void main(String[] args) {
String dbPath = "ip2region.xdb file path";
// 1、从 dbPath 加载整个 xdb 到内存。
byte[] cBuff;
try {
cBuff = Searcher.loadContentFromFile(dbPath);
} catch (Exception e) {
System.out.printf("failed to load content from `%s`: %s\n", dbPath, e);
return;
}
// 2、使用上述的 cBuff 创建一个完全基于内存的查询对象。
Searcher searcher;
try {
searcher = Searcher.newWithBuffer(cBuff);
} catch (Exception e) {
System.out.printf("failed to create content cached searcher: %s\n", e);
return;
}
// 3、查询
try {
String ip = "1.2.3.4";
long sTime = System.nanoTime();
String region = searcher.search(ip);
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
} catch (Exception e) {
System.out.printf("failed to search(%s): %s\n", ip, e);
}
// 4、关闭资源 - 该 searcher 对象可以安全用于并发，等整个服务关闭的时候再关闭 searcher
// searcher.close();
// 备注：并发使用，用整个 xdb 数据缓存创建的查询对象可以安全的用于并发，也就是你可以把这个 searcher 对象做成全局对象去跨线程访问。
}
}
``` ```
@ -174,7 +91,7 @@ options:
例如：使用默认的 data/ip2region.xdb 文件进行查询测试： 例如：使用默认的 data/ip2region.xdb 文件进行查询测试：
```bash ```bash
➜ java git:(v2.0_xdb) ✗ java -jar target/ip2region-2.6.0.jar search --db=../../data/ip2region.xdb $ java -jar target/ip2region-2.6.5.jar search --db=../../data/ip2region.xdb
ip2region xdb searcher test program, cachePolicy: vectorIndex ip2region xdb searcher test program, cachePolicy: vectorIndex
type 'quit' to exit type 'quit' to exit
ip2region>> 1.2.3.4 ip2region>> 1.2.3.4
@ -182,26 +99,31 @@ ip2region>> 1.2.3.4
ip2region>> ip2region>>
``` ```
输入 ip 即可进行查询测试，也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的查询效果。 输入 ip 即可进行查询测试，也可以分别设置 `--cache-policy` 来测试三种不同缓存实现的查询效果。
# bench 测试 # bench 测试
可以通过 `java -jar ip2region-{version}.jar bench` 命令来进行 bench 测试，一方面确保 `xdb` 文件没有错误，一方面可以评估查询性能： 可以通过 `java -jar ip2region-{version}.jar bench` 命令来进行 bench 测试，一方面确保 `xdb` 文件没有错误，一方面可以评估查询性能：
```bash ```bash
➜ java git:(v2.0_xdb) ✗ java -jar target/ip2region-2.6.0.jar bench java -jar target/ip2region-2.6.5.jar bench
java -jar ip2region-{version}.jar bench [command options] java -jar ip2region-{version}.jar bench [command options]
options: options:
--db string ip2region binary xdb file path --db string ip2region binary xdb file path
--src string source ip text file path --src string source ip text file path
--cache-policy string cache policy: file/vectorIndex/content --cache-policy string cache policy: file/vectorIndex/buffer
``` ```
例如：通过默认的 data/ip2region.xdb 和 data/ip.merge.txt 文件进行 bench 测试： 例如：通过默认的 data/ip2region.xdb 和 data/ip.merge.txt 文件进行 bench 测试：
```bash ```bash
➜ java git:(v2.0_xdb) ✗ java -jar target/ip2region-2.6.0.jar bench --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt $ java -jar target/ip2region-2.6.5.jar bench --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt --cache-policy=buffer
Bench finished, {cachePolicy: vectorIndex, total: 3417955, took: 8s, cost: 2 μs/op}
-- 笔者笔记本[mac 2.6 GHz 六核Intel Core i7]测试结果案例
Bench finished, {cachePolicy: buffer, total: 3417955, took: 2s, ioCount: 0, cost: 0 μs/op}
Bench finished, {cachePolicy: vectorIndex, total: 3417955, took: 28s, ioCount: 43465554, cost: 8 μs/op}
Bench finished, {cachePolicy: file, total: 3417955, took: 34s, ioCount: 43465554, cost: 9 μs/op}
``` ```
可以通过分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效果。 可以通过分别设置 `--cache-policy` 来测试三种不同缓存实现的效果。
@Note: 注意 bench 使用的 src 文件要是生成对应 xdb 文件相同的源文件。 @Note: 注意 bench 使用的 src 文件要是生成对应 xdb 文件相同的源文件。

28
binding/java/pom.xml
View File

@ -44,14 +44,38 @@
</properties> </properties>
<dependencies> <dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.24</version>
<scope>provided</scope>
</dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>
<artifactId>junit</artifactId> <artifactId>junit</artifactId>
<version>4.13.1</version> <version>4.13.2</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>net.jcip</groupId>
<artifactId>jcip-annotations</artifactId>
<version>1.0</version>
<scope>provided</scope>
</dependency>
</dependencies> </dependencies>
<repositories>
<repository>
<id>aliyun</id>
<url>https://maven.aliyun.com/repository/public</url>
</repository>
<repository>
<id>repo1</id>
<url>https://repo1.maven.org/maven2/</url>
</repository>
</repositories>
<build> <build>
<plugins> <plugins>
<plugin> <plugin>
@ -103,7 +127,7 @@
<configuration> <configuration>
<transformers> <transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.lionsoul.ip2region.SearchTest</mainClass> <mainClass>org.lionsoul.ip2region.xdb.SearchTest</mainClass>
</transformer> </transformer>
</transformers> </transformers>
</configuration> </configuration>

47
binding/java/src/main/java/org/lionsoul/ip2region/example/SearcherExample.java Normal file
View File

@ -0,0 +1,47 @@
package org.lionsoul.ip2region.example;
import org.lionsoul.ip2region.xdb.Searcher;
import java.util.concurrent.TimeUnit;
/**
* @see org.lionsoul.ip2region.xdb.Searcher
*/
public class SearcherExample {
public static void main(String[] args) throws Exception {
String dbPath = "ip2region.xdb file path";
// 基于文件查询，单次 search 需要文件 IO，非线程安全，性能最差
try (Searcher fileSearcher = Searcher.newWithFileOnly(dbPath)) {
searchExample(fileSearcher);
}
// 基于 VectorIndex 查询，单次 search 需要内存查询 + 文件 IO，非线程安全，性能次之
try (Searcher indexSearcher = Searcher.newWithVectorIndex(dbPath)) {
searchExample(indexSearcher);
}
// 推荐：基于 Buffer 查询，单次 search 全使用内存查询，线程安全，支持序列化，性能最佳
// 可重写 loadFile 方法扩展从其他文件系统(hdfs/s3/oss...)装载 buffer 数据
try (Searcher bufferSearcher = Searcher.newWithBuffer(dbPath)) {
searchExample(bufferSearcher);
}
// 备注：并发查询时，对于非线程安全实现推荐为每个线程创建一个独立的 searcher 对象单独使用。
}
private static void searchExample(Searcher searcher) {
String ip = "1.2.3.4";
try {
long sTime = System.nanoTime();
Searcher.Region region = searcher.searchRegion(ip);
long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime);
System.out.printf("{region: %s, searcher: %s, ioCount: %d, took: %d μs}\n",
region, searcher.getClass().getSimpleName(), region.getIoCount(), cost);
System.out.println(region.toRegionMsg());
} catch (Exception e) {
System.out.printf("failed to search(%s): %s\n", ip, e);
}
}
}

106
binding/java/src/main/java/org/lionsoul/ip2region/xdb/AbstractSearcher.java Normal file
View File

@ -0,0 +1,106 @@
package org.lionsoul.ip2region.xdb;
import java.nio.charset.StandardCharsets;
import static org.lionsoul.ip2region.xdb.InternalUtil.*;
/**
* @author Li.Wei by 2022/8/6
*/
public abstract class AbstractSearcher implements Searcher {
// constant defined copied from the xdb maker
protected static final int HEADER_INFO_LENGTH = 256;
protected static final int VECTOR_INDEX_ROWS = 256;
protected static final int VECTOR_INDEX_COLS = 256;
protected static final int VECTOR_INDEX_SIZE = 8;
protected static final int SEGMENT_INDEX_SIZE = 14;
@Deprecated
public int getIOCount() {
return 0;
}
public Region searchRegion(String ip) {
return search0(ip2long(ip));
}
public String search(String ip) {
return searchRegion(ip).getRegion();
}
private Region search0(long ip) {
int ioCount = 0;
// locate the segment index block based on the vector index
int il0 = (int) ((ip >> 24) & 0xFF);
int il1 = (int) ((ip >> 16) & 0xFF);
int idx = il0 * VECTOR_INDEX_COLS * VECTOR_INDEX_SIZE + il1 * VECTOR_INDEX_SIZE;
// System.out.printf("il0: %d, il1: %d, idx: %d\n", il0, il1, idx);
PointIndex pointIndex = this.pointIndex(idx);
int sPtr = pointIndex.sPtr, ePtr = pointIndex.ePtr;
// System.out.printf("sPtr: %d, ePtr: %d\n", sPtr, ePtr);
// binary search the segment index block to get the region info
final byte[] buff = new byte[SEGMENT_INDEX_SIZE];
int dataLen = -1, dataPtr = -1;
int l = 0, h = (ePtr - sPtr) / SEGMENT_INDEX_SIZE;
while (l <= h) {
int m = (l + h) >> 1;
int p = sPtr + m * SEGMENT_INDEX_SIZE;
// read the segment index
ioCount += read(p, buff);
long sip = getIntLong(buff, 0);
if (ip < sip) {
h = m - 1;
} else {
long eip = getIntLong(buff, 4);
if (ip > eip) {
l = m + 1;
} else {
dataLen = getInt2(buff, 8);
dataPtr = getInt(buff, 10);
break;
}
}
}
// empty match interception
// System.out.printf("dataLen: %d, dataPtr: %d\n", dataLen, dataPtr);
if (dataPtr < 0) {
return Region.builder().ioCount(ioCount).build();
}
// load and return the region data
final byte[] regionBuff = new byte[dataLen];
ioCount += read(dataPtr, regionBuff);
return Region.builder().region(new String(regionBuff, StandardCharsets.UTF_8)).ioCount(ioCount).build();
}
/**
* find index start,end point
*
* @param idx idx
* @return PointIndex
*/
protected abstract PointIndex pointIndex(int idx);
/**
* read 2 buffer
*
* @param offset offset
* @param buffer buffer
* @return io count
*/
protected abstract int read(int offset, byte[] buffer);
protected static class PointIndex {
int sPtr, ePtr;
public PointIndex(int sPtr, int ePtr) {
this.sPtr = sPtr;
this.ePtr = ePtr;
}
}
}

73
binding/java/src/main/java/org/lionsoul/ip2region/xdb/BufferSearcher.java Normal file
View File

@ -0,0 +1,73 @@
package org.lionsoul.ip2region.xdb;
import net.jcip.annotations.ThreadSafe;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.Serializable;
import static org.lionsoul.ip2region.xdb.InternalUtil.getInt;
/**
* xdb content buffer, used for in-memory search.
*/
@ThreadSafe
public class BufferSearcher extends AbstractSearcher implements Serializable {
private final byte[] contentBuff;
public BufferSearcher(byte[] contentBuff) {
this.contentBuff = contentBuff;
}
public BufferSearcher(String dbFile) {
try {
this.contentBuff = loadFile(dbFile);
} catch (IOException e) {
throw new SearcherException("load dbFile error: " + e.getMessage() + ", file=" + dbFile, e);
}
}
/**
* default load local file. override load hdfs/s3/oss...
*
* @param dbFile dbFile
* @return byte array
* @throws IOException ex
*/
protected byte[] loadFile(String dbFile) throws IOException {
return loadLocalFile(dbFile);
}
@Override
protected int read(int offset, byte[] buffer) {
System.arraycopy(contentBuff, offset, buffer, 0, buffer.length);
return 0;
}
@Override
protected PointIndex pointIndex(int idx) {
return new PointIndex(
getInt(contentBuff, HEADER_INFO_LENGTH + idx),
getInt(contentBuff, HEADER_INFO_LENGTH + idx + 4)
);
}
@Override
public void close() {
// nothing
}
private static byte[] loadLocalFile(String dbPath) throws IOException {
final byte[] buff;
try (RandomAccessFile handle = new RandomAccessFile(dbPath, "r")) {
handle.seek(0);
buff = new byte[(int) handle.length()];
int rLen = handle.read(buff);
if (rLen != buff.length) {
throw new IOException("incomplete read: read bytes should be " + buff.length);
}
}
return buff;
}
}

52
binding/java/src/main/java/org/lionsoul/ip2region/xdb/FileSearcher.java Normal file
View File

@ -0,0 +1,52 @@
package org.lionsoul.ip2region.xdb;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import static org.lionsoul.ip2region.xdb.InternalUtil.getInt;
/**
* random access file handle for file based search.
*/
public class FileSearcher extends AbstractSearcher {
private final RandomAccessFile handle;
public FileSearcher(String dbFile) {
try {
this.handle = new RandomAccessFile(dbFile, "r");
} catch (FileNotFoundException e) {
throw new SearcherException("load dbFile error: " + e.getMessage() + ", file=" + dbFile, e);
}
}
@Override
protected int read(int offset, byte[] buffer) {
int rLen;
try {
handle.seek(offset);
rLen = handle.read(buffer);
} catch (SearcherException | IOException e) {
throw new SearcherException("read error", e);
}
if (rLen != buffer.length) {
throw new SearcherException("incomplete read: read bytes should be " + buffer.length);
}
return 2;
}
@Override
protected PointIndex pointIndex(int idx) {
final byte[] buff = new byte[VECTOR_INDEX_SIZE];
read(HEADER_INFO_LENGTH + idx, buff);
return new PointIndex(
getInt(buff, 0),
getInt(buff, 4)
);
}
@Override
public void close() throws IOException {
this.handle.close();
}
}

30
binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java
View File

@ -7,9 +7,8 @@
package org.lionsoul.ip2region.xdb; package org.lionsoul.ip2region.xdb;
import java.awt.image.SampleModel; // Internal class
class Header {
public class Header {
public final int version; public final int version;
public final int indexPolicy; public final int indexPolicy;
public final int createdAt; public final int createdAt;
@ -19,21 +18,22 @@ public class Header {
public Header(byte[] buff) { public Header(byte[] buff) {
assert buff.length >= 16; assert buff.length >= 16;
version = Searcher.getInt2(buff, 0); version = InternalUtil.getInt2(buff, 0);
indexPolicy = Searcher.getInt2(buff, 2); indexPolicy = InternalUtil.getInt2(buff, 2);
createdAt = Searcher.getInt(buff, 4); createdAt = InternalUtil.getInt(buff, 4);
startIndexPtr = Searcher.getInt(buff, 8); startIndexPtr = InternalUtil.getInt(buff, 8);
endIndexPtr = Searcher.getInt(buff, 12); endIndexPtr = InternalUtil.getInt(buff, 12);
buffer = buff; buffer = buff;
} }
@Override public String toString() { @Override
public String toString() {
return "{" + return "{" +
"Version: " + version + ',' + "Version: " + version + ',' +
"IndexPolicy: " + indexPolicy + ',' + "IndexPolicy: " + indexPolicy + ',' +
"CreatedAt: " + createdAt + ',' + "CreatedAt: " + createdAt + ',' +
"StartIndexPtr: " + startIndexPtr + ',' + "StartIndexPtr: " + startIndexPtr + ',' +
"EndIndexPtr: " + endIndexPtr + "EndIndexPtr: " + endIndexPtr +
'}'; '}';
} }
} }

58
binding/java/src/main/java/org/lionsoul/ip2region/xdb/InternalUtil.java Normal file
View File

@ -0,0 +1,58 @@
package org.lionsoul.ip2region.xdb;
/**
* Internal Util
*/
public class InternalUtil {
private static final byte[] SHIFT_INDEX = {24, 16, 8, 0};
/* get an int from a byte array start from the specified offset */
protected static long getIntLong(byte[] b, int offset) {
return (
((b[offset++] & 0x000000FFL)) |
((b[offset++] << 8) & 0x0000FF00L) |
((b[offset++] << 16) & 0x00FF0000L) |
((b[offset] << 24) & 0xFF000000L)
);
}
protected static int getInt(byte[] b, int offset) {
return (
((b[offset++] & 0x000000FF)) |
((b[offset++] << 8) & 0x0000FF00) |
((b[offset++] << 16) & 0x00FF0000) |
((b[offset] << 24) & 0xFF000000)
);
}
protected static int getInt2(byte[] b, int offset) {
return (
(b[offset++] & 0x000000FF) |
(b[offset] & 0x0000FF00)
);
}
/* long int to ip string */
public static String long2ip(long ip) {
return String.valueOf((ip >> 24) & 0xFF) + '.' +
((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF);
}
/* check the specified ip address */
public static long ip2long(String ip) {
String[] ps = ip.split("\\.");
if (ps.length != 4) {
throw new SearcherException("invalid ip address `" + ip + "`");
}
long ipDst = 0;
for (int i = 0; i < ps.length; i++) {
int val = Integer.parseInt(ps[i]);
if (val > 255) {
throw new SearcherException("ip part `" + ps[i] + "` should be less then 256");
}
ipDst |= ((long) val << SHIFT_INDEX[i]);
}
return ipDst & 0xFFFFFFFFL;
}
}

138
binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java → binding/java/src/main/java/org/lionsoul/ip2region/xdb/SearchTest.java
View File

@ -4,16 +4,23 @@
// @Author Lion <chenxin619315@gmail.com> // @Author Lion <chenxin619315@gmail.com>
// @Date 2022/06/23 // @Date 2022/06/23
package org.lionsoul.ip2region; package org.lionsoul.ip2region.xdb;
import org.lionsoul.ip2region.xdb.Searcher; import java.io.BufferedReader;
import java.io.FileReader;
import java.io.*; import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
// for test
public class SearchTest { public class SearchTest {
public static void printHelp(String[] args) { protected static final String FILE = "file";
protected static final String VECTOR_INDEX = "vectorIndex";
protected static final String BUFFER = "buffer";
public static void printHelp() {
System.out.print("ip2region xdb searcher\n"); System.out.print("ip2region xdb searcher\n");
System.out.print("java -jar ip2region-{version}.jar [command] [command options]\n"); System.out.print("java -jar ip2region-{version}.jar [command] [command options]\n");
System.out.print("Command: \n"); System.out.print("Command: \n");
@ -21,22 +28,21 @@ public class SearchTest {
System.out.print(" bench search bench test\n"); System.out.print(" bench search bench test\n");
} }
public static Searcher createSearcher(String dbPath, String cachePolicy) throws IOException { public static AbstractSearcher createSearcher(String dbPath, String cachePolicy) throws IOException {
if ("file".equals(cachePolicy)) { switch (cachePolicy) {
return Searcher.newWithFileOnly(dbPath); case FILE:
} else if ("vectorIndex".equals(cachePolicy)) { return (AbstractSearcher) Searcher.newWithFileOnly(dbPath);
byte[] vIndex = Searcher.loadVectorIndexFromFile(dbPath); case VECTOR_INDEX:
return Searcher.newWithVectorIndex(dbPath, vIndex); return (AbstractSearcher) Searcher.newWithVectorIndex(dbPath);
} else if ("content".equals(cachePolicy)) { case BUFFER:
byte[] cBuff = Searcher.loadContentFromFile(dbPath); return (AbstractSearcher) Searcher.newWithBuffer(dbPath);
return Searcher.newWithBuffer(cBuff); default:
} else { throw new IOException("invalid cache policy `" + cachePolicy + "`, see help");
throw new IOException("invalid cache policy `" + cachePolicy + "`, options: file/vectorIndex/content");
} }
} }
public static void searchTest(String[] args) throws IOException { public static void searchTest(String[] args) throws Exception {
String dbPath = "", cachePolicy = "vectorIndex"; String dbPath = "", cachePolicy = VECTOR_INDEX;
for (final String r : args) { for (final String r : args) {
if (r.length() < 5) { if (r.length() < 5) {
continue; continue;
@ -74,16 +80,16 @@ public class SearchTest {
} }
Searcher searcher = createSearcher(dbPath, cachePolicy); Searcher searcher = createSearcher(dbPath, cachePolicy);
final BufferedReader reader = new BufferedReader(new InputStreamReader(System.in)); final BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8));
System.out.printf("ip2region xdb searcher test program, cachePolicy: %s\ntype 'quit' to exit\n", cachePolicy); System.out.printf("ip2region xdb searcher test program, cachePolicy: %s\ntype 'quit' to exit\n", cachePolicy);
while ( true ) { while (true) {
System.out.print("ip2region>> "); System.out.print("ip2region>> ");
String line = reader.readLine().trim(); String line = reader.readLine().trim();
if ( line.length() < 2 ) { if (line.length() < 2) {
continue; continue;
} }
if ( line.equalsIgnoreCase("quit") ) { if (line.equalsIgnoreCase("quit")) {
break; break;
} }
@ -91,9 +97,9 @@ public class SearchTest {
double sTime = System.nanoTime(); double sTime = System.nanoTime();
String region = searcher.search(line); String region = searcher.search(line);
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); System.out.printf("{region: %s, took: %d μs}\n", region, cost);
} catch (Exception e) { } catch (Exception e) {
System.out.printf("{err: %s, ioCount: %d}\n", e, searcher.getIOCount()); System.out.printf("{err: %s}\n", e);
} }
} }
@ -102,8 +108,8 @@ public class SearchTest {
System.out.println("searcher test program exited, thanks for trying"); System.out.println("searcher test program exited, thanks for trying");
} }
public static void benchTest(String[] args) throws IOException { public static void benchTest(String[] args) throws Exception {
String dbPath = "", srcPath = "", cachePolicy = "vectorIndex"; String dbPath = "", srcPath = "", cachePolicy = VECTOR_INDEX;
for (final String r : args) { for (final String r : args) {
if (r.length() < 5) { if (r.length() < 5) {
continue; continue;
@ -121,15 +127,19 @@ public class SearchTest {
String key = r.substring(2, sIdx); String key = r.substring(2, sIdx);
String val = r.substring(sIdx + 1); String val = r.substring(sIdx + 1);
if ("db".equals(key)) { switch (key) {
dbPath = val; case "db":
} else if ("src".equals(key)) { dbPath = val;
srcPath = val; break;
} else if ("cache-policy".equals(key)) { case "src":
cachePolicy = val; srcPath = val;
} else { break;
System.out.printf("undefined option `%s`\n", r); case "cache-policy":
return; cachePolicy = val;
break;
default:
System.out.printf("undefined option `%s`\n", r);
return;
} }
} }
@ -138,12 +148,13 @@ public class SearchTest {
System.out.print("options:\n"); System.out.print("options:\n");
System.out.print(" --db string ip2region binary xdb file path\n"); System.out.print(" --db string ip2region binary xdb file path\n");
System.out.print(" --src string source ip text file path\n"); System.out.print(" --src string source ip text file path\n");
System.out.print(" --cache-policy string cache policy: file/vectorIndex/content\n"); System.out.print(" --cache-policy string cache policy: " + FILE + "/" + VECTOR_INDEX + "/" + BUFFER + "\n");
return; return;
} }
Searcher searcher = createSearcher(dbPath, cachePolicy); AbstractSearcher searcher = createSearcher(dbPath, cachePolicy);
long count = 0, costs = 0, tStart = System.nanoTime(); long count = 0, costs = 0, ioCount = 0, tStart = System.nanoTime();
String line; String line;
final BufferedReader reader = new BufferedReader(new FileReader(srcPath)); final BufferedReader reader = new BufferedReader(new FileReader(srcPath));
while ((line = reader.readLine()) != null) { while ((line = reader.readLine()) != null) {
@ -156,7 +167,7 @@ public class SearchTest {
long sip; long sip;
try { try {
sip = Searcher.checkIP(ps[0]); sip = InternalUtil.ip2long(ps[0]);
} catch (Exception e) { } catch (Exception e) {
System.out.printf("check start ip `%s`: %s\n", ps[0], e); System.out.printf("check start ip `%s`: %s\n", ps[0], e);
return; return;
@ -164,7 +175,7 @@ public class SearchTest {
long eip; long eip;
try { try {
eip = Searcher.checkIP(ps[1]); eip = InternalUtil.ip2long(ps[1]);
} catch (Exception e) { } catch (Exception e) {
System.out.printf("check end ip `%s`: %s\n", ps[1], e); System.out.printf("check end ip `%s`: %s\n", ps[1], e);
return; return;
@ -178,12 +189,12 @@ public class SearchTest {
long mip = (sip + eip) >> 1; long mip = (sip + eip) >> 1;
for (final long ip : new long[]{sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) { for (final long ip : new long[]{sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) {
long sTime = System.nanoTime(); long sTime = System.nanoTime();
String region = searcher.search(ip); Searcher.Region region = searcher.searchRegion(InternalUtil.long2ip(ip));
costs += System.nanoTime() - sTime; costs += System.nanoTime() - sTime;
ioCount += region.getIoCount();
// check the region info // check the region info
if (!ps[2].equals(region)) { if (!ps[2].equals(region.getRegion())) {
System.out.printf("failed search(%s) with (%s != %s)\n", Searcher.long2ip(ip), region, ps[2]); System.out.printf("failed search(%s) with (%s != %s)\n", InternalUtil.long2ip(ip), region, ps[2]);
return; return;
} }
@ -194,31 +205,36 @@ public class SearchTest {
reader.close(); reader.close();
searcher.close(); searcher.close();
long took = System.nanoTime() - tStart; long took = System.nanoTime() - tStart;
System.out.printf("Bench finished, {cachePolicy: %s, total: %d, took: %ds, cost: %d μs/op}\n", System.out.printf("Bench finished, {cachePolicy: %s, total: %d, took: %ds, ioCount: %d, cost: %d μs/op}\n",
cachePolicy, count, TimeUnit.NANOSECONDS.toSeconds(took), cachePolicy, count, TimeUnit.NANOSECONDS.toSeconds(took),
count == 0 ? 0 : TimeUnit.NANOSECONDS.toMicros(costs/count)); ioCount,
count == 0 ? 0 : TimeUnit.NANOSECONDS.toMicros(costs / count));
} }
public static void main(String[] args) { public static void main(String[] args) {
if (args.length < 1) { if (args.length < 1) {
printHelp(args); printHelp();
return; return;
} }
if ("search".equals(args[0])) { switch (args[0]) {
try { case "search":
searchTest(args); try {
} catch (IOException e) { searchTest(args);
System.out.printf("failed running search test: %s\n", e); } catch (Exception e) {
} System.out.printf("failed running search test: %s\n", e);
} else if ("bench".equals(args[0])) { }
try { break;
benchTest(args); case "bench":
} catch (IOException e) { try {
System.out.printf("failed running bench test: %s\n", e); benchTest(args);
} } catch (Exception e) {
} else { System.out.printf("failed running bench test: %s\n", e);
printHelp(args); }
break;
default:
printHelp();
break;
} }
} }

292
binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java
View File

@ -9,264 +9,76 @@ package org.lionsoul.ip2region.xdb;
// @Date 2022/06/23 // @Date 2022/06/23
import java.io.IOException; import lombok.Builder;
import java.io.RandomAccessFile; import lombok.Data;
public class Searcher { /**
// constant defined copied from the xdb maker * @see FileSearcher
public static final int HeaderInfoLength = 256; * @see VectorIndexSearcher
public static final int VectorIndexRows = 256; * @see BufferSearcher
public static final int VectorIndexCols = 256; */
public static final int VectorIndexSize = 8; public interface Searcher extends AutoCloseable {
public static final int SegmentIndexSize = 14;
// random access file handle for file based search // FileSearcher
private final RandomAccessFile handle; static Searcher newWithFileOnly(String dbPath) {
return new FileSearcher(dbPath);
private int ioCount = 0;
// vector index.
// use the byte[] instead of VectorIndex entry array to keep
// the minimal memory allocation.
private final byte[] vectorIndex;
// xdb content buffer, used for in-memory search
private final byte[] contentBuff;
// --- static method to create searchers
public static Searcher newWithFileOnly(String dbPath) throws IOException {
return new Searcher(dbPath, null, null);
} }
public static Searcher newWithVectorIndex(String dbPath, byte[] vectorIndex) throws IOException { // IndexSearcher
return new Searcher(dbPath, vectorIndex, null); static Searcher newWithVectorIndex(String dbPath) {
return new VectorIndexSearcher(dbPath);
} }
public static Searcher newWithBuffer(byte[] cBuff) throws IOException { // IndexSearcher
return new Searcher(null, null, cBuff); static Searcher newWithVectorIndex(String dbPath, byte[] vectorIndex) {
return new VectorIndexSearcher(dbPath, vectorIndex);
} }
// --- End of creator // BufferSearcher
static Searcher newWithBuffer(String dbPath) {
public Searcher(String dbFile, byte[] vectorIndex, byte[] cBuff) throws IOException { return new BufferSearcher(dbPath);
if (cBuff != null) {
this.handle = null;
this.vectorIndex = null;
this.contentBuff = cBuff;
} else {
this.handle = new RandomAccessFile(dbFile, "r");
this.vectorIndex = vectorIndex;
this.contentBuff = null;
}
} }
public void close() throws IOException { // BufferSearcher
if (this.handle != null) { static Searcher newWithBuffer(byte[] cBuff) {
this.handle.close(); return new BufferSearcher(cBuff);
}
} }
public int getIOCount() { String search(String ip);
return ioCount;
}
public String search(String ipStr) throws Exception { Region searchRegion(String ip);
long ip = checkIP(ipStr);
return search(ip);
}
public String search(long ip) throws IOException { @Data
// reset the global counter @Builder
this.ioCount = 0; class Region {
private int ioCount;
private String region;
// locate the segment index block based on the vector index public RegionMsg toRegionMsg() {
int sPtr = 0, ePtr = 0; if (region == null) {
int il0 = (int) ((ip >> 24) & 0xFF); return null;
int il1 = (int) ((ip >> 16) & 0xFF);
int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
// System.out.printf("il0: %d, il1: %d, idx: %d\n", il0, il1, idx);
if (vectorIndex != null) {
sPtr = getInt(vectorIndex, idx);
ePtr = getInt(vectorIndex, idx + 4);
} else if (contentBuff != null) {
sPtr = getInt(contentBuff, HeaderInfoLength + idx);
ePtr = getInt(contentBuff, HeaderInfoLength + idx + 4);
} else {
final byte[] buff = new byte[VectorIndexSize];
read(HeaderInfoLength + idx, buff);
sPtr = getInt(buff, 0);
ePtr = getInt(buff, 4);
}
// System.out.printf("sPtr: %d, ePtr: %d\n", sPtr, ePtr);
// binary search the segment index block to get the region info
final byte[] buff = new byte[SegmentIndexSize];
int dataLen = -1, dataPtr = -1;
int l = 0, h = (ePtr - sPtr) / SegmentIndexSize;
while (l <= h) {
int m = (l + h) >> 1;
int p = sPtr + m * SegmentIndexSize;
// read the segment index
read(p, buff);
long sip = getIntLong(buff, 0);
if (ip < sip) {
h = m - 1;
} else {
long eip = getIntLong(buff, 4);
if (ip > eip) {
l = m + 1;
} else {
dataLen = getInt2(buff, 8);
dataPtr = getInt(buff, 10);
break;
}
}
}
// empty match interception
// System.out.printf("dataLen: %d, dataPtr: %d\n", dataLen, dataPtr);
if (dataPtr < 0) {
return null;
}
// load and return the region data
final byte[] regionBuff = new byte[dataLen];
read(dataPtr, regionBuff);
return new String(regionBuff, "utf-8");
}
protected void read(int offset, byte[] buffer) throws IOException {
// check the in-memory buffer first
if (contentBuff != null) {
// @TODO: reduce data copying, directly decode the data ?
System.arraycopy(contentBuff, offset, buffer, 0, buffer.length);
return;
}
// read from the file handle
assert handle != null;
handle.seek(offset);
this.ioCount++;
int rLen = handle.read(buffer);
if (rLen != buffer.length) {
throw new IOException("incomplete read: read bytes should be " + buffer.length);
}
}
// --- static cache util function
public static Header loadHeader(RandomAccessFile handle) throws IOException {
handle.seek(0);
final byte[] buff = new byte[HeaderInfoLength];
handle.read(buff);
return new Header(buff);
}
public static Header loadHeaderFromFile(String dbPath) throws IOException {
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
final Header header = loadHeader(handle);
handle.close();
return header;
}
public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException {
handle.seek(HeaderInfoLength);
int len = VectorIndexRows * VectorIndexCols * SegmentIndexSize;
final byte[] buff = new byte[len];
int rLen = handle.read(buff);
if (rLen != len) {
throw new IOException("incomplete read: read bytes should be " + len);
}
return buff;
}
public static byte[] loadVectorIndexFromFile(String dbPath) throws IOException {
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
final byte[] vIndex = loadVectorIndex(handle);
handle.close();
return vIndex;
}
public static byte[] loadContent(RandomAccessFile handle) throws IOException {
handle.seek(0);
final byte[] buff = new byte[(int) handle.length()];
int rLen = handle.read(buff);
if (rLen != buff.length) {
throw new IOException("incomplete read: read bytes should be " + buff.length);
}
return buff;
}
public static byte[] loadContentFromFile(String dbPath) throws IOException {
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
final byte[] content = loadContent(handle);
handle.close();
return content;
}
// --- End cache load util function
// --- static util method
/* get an int from a byte array start from the specified offset */
public static long getIntLong(byte[] b, int offset) {
return (
((b[offset++] & 0x000000FFL)) |
((b[offset++] << 8) & 0x0000FF00L) |
((b[offset++] << 16) & 0x00FF0000L) |
((b[offset ] << 24) & 0xFF000000L)
);
}
public static int getInt(byte[] b, int offset) {
return (
((b[offset++] & 0x000000FF)) |
((b[offset++] << 8) & 0x0000FF00) |
((b[offset++] << 16) & 0x00FF0000) |
((b[offset ] << 24) & 0xFF000000)
);
}
public static int getInt2(byte[] b, int offset) {
return (
(b[offset++] & 0x000000FF) |
(b[offset ] & 0x0000FF00)
);
}
/* long int to ip string */
public static String long2ip( long ip )
{
return String.valueOf((ip >> 24) & 0xFF) + '.' +
((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF);
}
public static final byte[] shiftIndex = {24, 16, 8, 0};
/* check the specified ip address */
public static long checkIP(String ip) throws Exception {
String[] ps = ip.split("\\.");
if (ps.length != 4) {
throw new Exception("invalid ip address `" + ip + "`");
}
long ipDst = 0;
for (int i = 0; i < ps.length; i++) {
int val = Integer.parseInt(ps[i]);
if (val > 255) {
throw new Exception("ip part `"+ps[i]+"` should be less then 256");
} }
ipDst |= ((long) val << shiftIndex[i]); final String[] ss = region.split("\\|");
return RegionMsg.builder()
.country(ss[0])
.region(ss[1])
.province(ss[2])
.city(ss[3])
.isp(ss[4])
.build();
} }
return ipDst & 0xFFFFFFFFL;
} }
// Country|Region|Province|City|ISP
// 国家|区域|省份|城市|ISP
@Data
@Builder
class RegionMsg {
private String country;
private String region;
private String province;
private String city;
private String isp;
}
} }

25
binding/java/src/main/java/org/lionsoul/ip2region/xdb/SearcherException.java Normal file
View File

@ -0,0 +1,25 @@
package org.lionsoul.ip2region.xdb;
/**
* Searcher RuntimeException
*/
public class SearcherException extends RuntimeException {
public SearcherException() {
}
public SearcherException(String message) {
super(message);
}
public SearcherException(String message, Throwable cause) {
super(message, cause);
}
public SearcherException(Throwable cause) {
super(cause);
}
public SearcherException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}

9
binding/java/src/main/java/org/lionsoul/ip2region/UtilTest.java → binding/java/src/main/java/org/lionsoul/ip2region/xdb/UtilTest.java
View File

@ -4,17 +4,16 @@
// @Author Lion <chenxin619315@gmail.com> // @Author Lion <chenxin619315@gmail.com>
// @Date 2022/06/23 // @Date 2022/06/23
package org.lionsoul.ip2region; package org.lionsoul.ip2region.xdb;
import org.lionsoul.ip2region.xdb.Searcher;
// for test
public class UtilTest { public class UtilTest {
public static void testIP2Long() { public static void testIP2Long() {
String ip = "1.2.3.4"; String ip = "1.2.3.4";
long ipAddr = 0; long ipAddr = 0;
try { try {
ipAddr = Searcher.checkIP(ip); ipAddr = InternalUtil.ip2long(ip);
} catch (Exception e) { } catch (Exception e) {
System.out.printf("failed to check ip: %s\n", e); System.out.printf("failed to check ip: %s\n", e);
return; return;
@ -25,7 +24,7 @@ public class UtilTest {
return; return;
} }
String ip2 = Searcher.long2ip(ipAddr); String ip2 = InternalUtil.long2ip(ipAddr);
if (!ip.equals(ip2)) { if (!ip.equals(ip2)) {
System.out.print("failed long2ip\n"); System.out.print("failed long2ip\n");
return; return;

52
binding/java/src/main/java/org/lionsoul/ip2region/xdb/VectorIndexSearcher.java Normal file
View File

@ -0,0 +1,52 @@
package org.lionsoul.ip2region.xdb;
import java.io.IOException;
import java.io.RandomAccessFile;
import static org.lionsoul.ip2region.xdb.InternalUtil.getInt;
/**
* vector index.
* use the byte[] instead of VectorIndex entry array to keep the minimal memory allocation.
*/
public class VectorIndexSearcher extends FileSearcher {
private final byte[] vectorIndex;
public VectorIndexSearcher(String dbFile) {
this(dbFile, null);
}
public VectorIndexSearcher(String dbFile, byte[] vectorIndex) {
super(dbFile);
try {
this.vectorIndex = vectorIndex != null ? vectorIndex : loadVectorIndexFromFile(dbFile);
} catch (SearcherException | IOException e) {
throw new SearcherException("load dbFile error: " + e.getMessage() + ", file=" + dbFile, e);
}
}
@Override
protected PointIndex pointIndex(int idx) {
return new PointIndex(
getInt(vectorIndex, idx),
getInt(vectorIndex, idx + 4)
);
}
// --- static util function
private static byte[] loadVectorIndexFromFile(String dbPath) throws IOException {
final byte[] buff;
try (RandomAccessFile handle = new RandomAccessFile(dbPath, "r")) {
handle.seek(HEADER_INFO_LENGTH);
int len = VECTOR_INDEX_ROWS * VECTOR_INDEX_COLS * SEGMENT_INDEX_SIZE;
buff = new byte[len];
int rLen = handle.read(buff);
if (rLen != len) {
throw new IOException("incomplete read: read bytes should be " + len);
}
}
return buff;
}
}