🎨: building\java 中 Searcher 实现 Closeable 接口
This commit is contained in:
parent
6ed8bf0118
commit
fb8e0ad9c9
|
|
@ -15,35 +15,29 @@
|
||||||
|
|
||||||
```java
|
```java
|
||||||
import org.lionsoul.ip2region.xdb.Searcher;
|
import org.lionsoul.ip2region.xdb.Searcher;
|
||||||
|
|
||||||
import java.io.*;
|
import java.io.*;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
public class SearcherTest {
|
public class SearcherTest {
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
// 1、创建 searcher 对象
|
|
||||||
String dbPath = "ip2region.xdb file path";
|
String dbPath = "ip2region.xdb file path";
|
||||||
Searcher searcher = null;
|
String ip = "1.2.3.4";
|
||||||
try {
|
|
||||||
searcher = Searcher.newWithFileOnly(dbPath);
|
|
||||||
} catch (IOException e) {
|
|
||||||
System.out.printf("failed to create searcher with `%s`: %s\n", dbPath, e);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2、查询
|
// 1、创建 searcher 对象
|
||||||
try {
|
try (Searcher searcher = Searcher.newWithFileOnly(dbPath)) {
|
||||||
String ip = "1.2.3.4";
|
// 2、查询
|
||||||
long sTime = System.nanoTime();
|
long sTime = System.nanoTime();
|
||||||
String region = searcher.search(ip);
|
String region = searcher.search(ip);
|
||||||
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
|
long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime);
|
||||||
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
|
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
|
||||||
|
} catch (IOException e) {
|
||||||
|
System.out.printf("failed to create searcher with `%s`: %s\n", dbPath, e);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
System.out.printf("failed to search(%s): %s\n", ip, e);
|
System.out.printf("failed to search(%s): %s\n", ip, e);
|
||||||
}
|
}
|
||||||
|
// 3、关闭资源(这里通过 try-with-resources 自动关闭)
|
||||||
|
|
||||||
// 3、关闭资源
|
|
||||||
searcher.close();
|
|
||||||
|
|
||||||
// 备注:并发使用,每个线程需要创建一个独立的 searcher 对象单独使用。
|
// 备注:并发使用,每个线程需要创建一个独立的 searcher 对象单独使用。
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -54,12 +48,14 @@ public class SearcherTest {
|
||||||
我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据,然后全局缓存,每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作,从而加速查询,减少 IO 压力。
|
我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据,然后全局缓存,每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作,从而加速查询,减少 IO 压力。
|
||||||
```java
|
```java
|
||||||
import org.lionsoul.ip2region.xdb.Searcher;
|
import org.lionsoul.ip2region.xdb.Searcher;
|
||||||
|
|
||||||
import java.io.*;
|
import java.io.*;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
public class SearcherTest {
|
public class SearcherTest {
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
String dbPath = "ip2region.xdb file path";
|
String dbPath = "ip2region.xdb file path";
|
||||||
|
String ip = "1.2.3.4";
|
||||||
|
|
||||||
// 1、从 dbPath 中预先加载 VectorIndex 缓存,并且把这个得到的数据作为全局变量,后续反复使用。
|
// 1、从 dbPath 中预先加载 VectorIndex 缓存,并且把这个得到的数据作为全局变量,后续反复使用。
|
||||||
byte[] vIndex;
|
byte[] vIndex;
|
||||||
|
|
@ -71,27 +67,17 @@ public class SearcherTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2、使用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。
|
// 2、使用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。
|
||||||
Searcher searcher;
|
try (Searcher searcher = Searcher.newWithVectorIndex(dbPath, vIndex)) {
|
||||||
try {
|
|
||||||
searcher = Searcher.newWithVectorIndex(dbPath, vIndex);
|
|
||||||
} catch (Exception e) {
|
|
||||||
System.out.printf("failed to create vectorIndex cached searcher with `%s`: %s\n", dbPath, e);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3、查询
|
|
||||||
try {
|
|
||||||
String ip = "1.2.3.4";
|
|
||||||
long sTime = System.nanoTime();
|
long sTime = System.nanoTime();
|
||||||
String region = searcher.search(ip);
|
String region = searcher.search(ip);
|
||||||
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
|
long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime);
|
||||||
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
|
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
|
||||||
|
} catch (IOException e) {
|
||||||
|
System.out.printf("failed to create vectorIndex cached searcher with `%s`: %s\n", dbPath, e);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
System.out.printf("failed to search(%s): %s\n", ip, e);
|
System.out.printf("failed to search(%s): %s\n", ip, e);
|
||||||
}
|
}
|
||||||
|
// 3、关闭资源(这里通过 try-with-resources 自动关闭)
|
||||||
// 4、关闭资源
|
|
||||||
searcher.close();
|
|
||||||
|
|
||||||
// 备注:每个线程需要单独创建一个独立的 Searcher 对象,但是都共享全局的制度 vIndex 缓存。
|
// 备注:每个线程需要单独创建一个独立的 Searcher 对象,但是都共享全局的制度 vIndex 缓存。
|
||||||
}
|
}
|
||||||
|
|
@ -103,12 +89,14 @@ public class SearcherTest {
|
||||||
我们也可以预先加载整个 ip2region.xdb 的数据到内存,然后基于这个数据创建查询对象来实现完全基于文件的查询,类似之前的 memory search。
|
我们也可以预先加载整个 ip2region.xdb 的数据到内存,然后基于这个数据创建查询对象来实现完全基于文件的查询,类似之前的 memory search。
|
||||||
```java
|
```java
|
||||||
import org.lionsoul.ip2region.xdb.Searcher;
|
import org.lionsoul.ip2region.xdb.Searcher;
|
||||||
|
|
||||||
import java.io.*;
|
import java.io.*;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
public class SearcherTest {
|
public class SearcherTest {
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
String dbPath = "ip2region.xdb file path";
|
String dbPath = "ip2region.xdb file path";
|
||||||
|
String ip = "1.2.3.4";
|
||||||
|
|
||||||
// 1、从 dbPath 加载整个 xdb 到内存。
|
// 1、从 dbPath 加载整个 xdb 到内存。
|
||||||
byte[] cBuff;
|
byte[] cBuff;
|
||||||
|
|
@ -123,22 +111,21 @@ public class SearcherTest {
|
||||||
Searcher searcher;
|
Searcher searcher;
|
||||||
try {
|
try {
|
||||||
searcher = Searcher.newWithBuffer(cBuff);
|
searcher = Searcher.newWithBuffer(cBuff);
|
||||||
} catch (Exception e) {
|
} catch (IOException e) {
|
||||||
System.out.printf("failed to create content cached searcher: %s\n", e);
|
System.out.printf("failed to create content cached searcher: %s\n", e);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3、查询
|
// 3、查询
|
||||||
try {
|
try {
|
||||||
String ip = "1.2.3.4";
|
|
||||||
long sTime = System.nanoTime();
|
long sTime = System.nanoTime();
|
||||||
String region = searcher.search(ip);
|
String region = searcher.search(ip);
|
||||||
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
|
long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime);
|
||||||
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
|
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
System.out.printf("failed to search(%s): %s\n", ip, e);
|
System.out.printf("failed to search(%s): %s\n", ip, e);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4、关闭资源 - 该 searcher 对象可以安全用于并发,等整个服务关闭的时候再关闭 searcher
|
// 4、关闭资源 - 该 searcher 对象可以安全用于并发,等整个服务关闭的时候再关闭 searcher
|
||||||
// searcher.close();
|
// searcher.close();
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -41,6 +41,7 @@
|
||||||
<properties>
|
<properties>
|
||||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||||
|
<java.version>1.8</java.version>
|
||||||
<maven.compiler.source>1.6</maven.compiler.source>
|
<maven.compiler.source>1.6</maven.compiler.source>
|
||||||
<maven.compiler.target>1.6</maven.compiler.target>
|
<maven.compiler.target>1.6</maven.compiler.target>
|
||||||
</properties>
|
</properties>
|
||||||
|
|
@ -112,6 +113,15 @@
|
||||||
</execution>
|
</execution>
|
||||||
</executions>
|
</executions>
|
||||||
</plugin>
|
</plugin>
|
||||||
|
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<source>${java.version}</source>
|
||||||
|
<target>${java.version}</target>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
</plugins>
|
</plugins>
|
||||||
</build>
|
</build>
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,10 @@ package org.lionsoul.ip2region;
|
||||||
|
|
||||||
import org.lionsoul.ip2region.xdb.Searcher;
|
import org.lionsoul.ip2region.xdb.Searcher;
|
||||||
|
|
||||||
import java.io.*;
|
import java.io.BufferedReader;
|
||||||
|
import java.io.FileReader;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStreamReader;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
public class SearchTest {
|
public class SearchTest {
|
||||||
|
|
@ -65,7 +68,7 @@ public class SearchTest {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (dbPath.length() < 1) {
|
if (dbPath.isEmpty()) {
|
||||||
System.out.print("java -jar ip2region-{version}.jar search [command options]\n");
|
System.out.print("java -jar ip2region-{version}.jar search [command options]\n");
|
||||||
System.out.print("options:\n");
|
System.out.print("options:\n");
|
||||||
System.out.print(" --db string ip2region binary xdb file path\n");
|
System.out.print(" --db string ip2region binary xdb file path\n");
|
||||||
|
|
@ -73,32 +76,30 @@ public class SearchTest {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
Searcher searcher = createSearcher(dbPath, cachePolicy);
|
try (Searcher searcher = createSearcher(dbPath, cachePolicy);
|
||||||
final BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
|
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in))) {
|
||||||
System.out.printf("ip2region xdb searcher test program, cachePolicy: %s\ntype 'quit' to exit\n", cachePolicy);
|
System.out.printf("ip2region xdb searcher test program, cachePolicy: %s\ntype 'quit' to exit\n", cachePolicy);
|
||||||
while ( true ) {
|
while (true) {
|
||||||
System.out.print("ip2region>> ");
|
System.out.print("ip2region>> ");
|
||||||
String line = reader.readLine().trim();
|
String line = reader.readLine().trim();
|
||||||
if ( line.length() < 2 ) {
|
if (line.length() < 2) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if ( line.equalsIgnoreCase("quit") ) {
|
if ("quit".equalsIgnoreCase(line)) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
double sTime = System.nanoTime();
|
double sTime = System.nanoTime();
|
||||||
String region = searcher.search(line);
|
String region = searcher.search(line);
|
||||||
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
|
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
|
||||||
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
|
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
System.out.printf("{err: %s, ioCount: %d}\n", e, searcher.getIOCount());
|
System.out.printf("{err: %s, ioCount: %d}\n", e, searcher.getIOCount());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
reader.close();
|
|
||||||
searcher.close();
|
|
||||||
System.out.println("searcher test program exited, thanks for trying");
|
System.out.println("searcher test program exited, thanks for trying");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -121,19 +122,23 @@ public class SearchTest {
|
||||||
|
|
||||||
String key = r.substring(2, sIdx);
|
String key = r.substring(2, sIdx);
|
||||||
String val = r.substring(sIdx + 1);
|
String val = r.substring(sIdx + 1);
|
||||||
if ("db".equals(key)) {
|
switch (key) {
|
||||||
dbPath = val;
|
case "db":
|
||||||
} else if ("src".equals(key)) {
|
dbPath = val;
|
||||||
srcPath = val;
|
break;
|
||||||
} else if ("cache-policy".equals(key)) {
|
case "src":
|
||||||
cachePolicy = val;
|
srcPath = val;
|
||||||
} else {
|
break;
|
||||||
System.out.printf("undefined option `%s`\n", r);
|
case "cache-policy":
|
||||||
return;
|
cachePolicy = val;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
System.out.printf("undefined option `%s`\n", r);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (dbPath.length() < 1 || srcPath.length() < 1) {
|
if (dbPath.isEmpty() || srcPath.isEmpty()) {
|
||||||
System.out.print("java -jar ip2region-{version}.jar bench [command options]\n");
|
System.out.print("java -jar ip2region-{version}.jar bench [command options]\n");
|
||||||
System.out.print("options:\n");
|
System.out.print("options:\n");
|
||||||
System.out.print(" --db string ip2region binary xdb file path\n");
|
System.out.print(" --db string ip2region binary xdb file path\n");
|
||||||
|
|
@ -142,61 +147,59 @@ public class SearchTest {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
Searcher searcher = createSearcher(dbPath, cachePolicy);
|
|
||||||
long count = 0, costs = 0, tStart = System.nanoTime();
|
long count = 0, costs = 0, tStart = System.nanoTime();
|
||||||
String line;
|
try (Searcher searcher = createSearcher(dbPath, cachePolicy);
|
||||||
final BufferedReader reader = new BufferedReader(new FileReader(srcPath));
|
BufferedReader reader = new BufferedReader(new FileReader(srcPath))) {
|
||||||
while ((line = reader.readLine()) != null) {
|
String line;
|
||||||
String l = line.trim();
|
while ((line = reader.readLine()) != null) {
|
||||||
String[] ps = l.split("\\|", 3);
|
String l = line.trim();
|
||||||
if (ps.length != 3) {
|
String[] ps = l.split("\\|", 3);
|
||||||
System.out.printf("invalid ip segment `%s`\n", l);
|
if (ps.length != 3) {
|
||||||
return;
|
System.out.printf("invalid ip segment `%s`\n", l);
|
||||||
}
|
|
||||||
|
|
||||||
long sip;
|
|
||||||
try {
|
|
||||||
sip = Searcher.checkIP(ps[0]);
|
|
||||||
} catch (Exception e) {
|
|
||||||
System.out.printf("check start ip `%s`: %s\n", ps[0], e);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
long eip;
|
|
||||||
try {
|
|
||||||
eip = Searcher.checkIP(ps[1]);
|
|
||||||
} catch (Exception e) {
|
|
||||||
System.out.printf("check end ip `%s`: %s\n", ps[1], e);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (sip > eip) {
|
|
||||||
System.out.printf("start ip(%s) should not be greater than end ip(%s)\n", ps[0], ps[1]);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
long mip = (sip + eip) >> 1;
|
|
||||||
for (final long ip : new long[]{sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) {
|
|
||||||
long sTime = System.nanoTime();
|
|
||||||
String region = searcher.search(ip);
|
|
||||||
costs += System.nanoTime() - sTime;
|
|
||||||
|
|
||||||
// check the region info
|
|
||||||
if (!ps[2].equals(region)) {
|
|
||||||
System.out.printf("failed search(%s) with (%s != %s)\n", Searcher.long2ip(ip), region, ps[2]);
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
count++;
|
long sip;
|
||||||
|
try {
|
||||||
|
sip = Searcher.checkIP(ps[0]);
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.printf("check start ip `%s`: %s\n", ps[0], e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
long eip;
|
||||||
|
try {
|
||||||
|
eip = Searcher.checkIP(ps[1]);
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.printf("check end ip `%s`: %s\n", ps[1], e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sip > eip) {
|
||||||
|
System.out.printf("start ip(%s) should not be greater than end ip(%s)\n", ps[0], ps[1]);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
long mip = (sip + eip) >> 1;
|
||||||
|
for (final long ip : new long[]{sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) {
|
||||||
|
long sTime = System.nanoTime();
|
||||||
|
String region = searcher.search(ip);
|
||||||
|
costs += System.nanoTime() - sTime;
|
||||||
|
|
||||||
|
// check the region info
|
||||||
|
if (!ps[2].equals(region)) {
|
||||||
|
System.out.printf("failed search(%s) with (%s != %s)\n", Searcher.long2ip(ip), region, ps[2]);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
count++;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
reader.close();
|
|
||||||
searcher.close();
|
|
||||||
long took = System.nanoTime() - tStart;
|
long took = System.nanoTime() - tStart;
|
||||||
System.out.printf("Bench finished, {cachePolicy: %s, total: %d, took: %ds, cost: %d μs/op}\n",
|
System.out.printf("Bench finished, {cachePolicy: %s, total: %d, took: %ds, cost: %d μs/op}\n",
|
||||||
cachePolicy, count, TimeUnit.NANOSECONDS.toSeconds(took),
|
cachePolicy, count, TimeUnit.NANOSECONDS.toSeconds(took),
|
||||||
count == 0 ? 0 : TimeUnit.NANOSECONDS.toMicros(costs/count));
|
count == 0 ? 0 : TimeUnit.NANOSECONDS.toMicros(costs / count));
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ public class UtilTest {
|
||||||
|
|
||||||
public static void testIP2Long() {
|
public static void testIP2Long() {
|
||||||
String ip = "1.2.3.4";
|
String ip = "1.2.3.4";
|
||||||
long ipAddr = 0;
|
long ipAddr;
|
||||||
try {
|
try {
|
||||||
ipAddr = Searcher.checkIP(ip);
|
ipAddr = Searcher.checkIP(ip);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
|
|
||||||
|
|
@ -24,13 +24,14 @@ public class Header {
|
||||||
buffer = buff;
|
buffer = buff;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override public String toString() {
|
@Override
|
||||||
|
public String toString() {
|
||||||
return "{" +
|
return "{" +
|
||||||
"Version: " + version + ',' +
|
"Version: " + version + ',' +
|
||||||
"IndexPolicy: " + indexPolicy + ',' +
|
"IndexPolicy: " + indexPolicy + ',' +
|
||||||
"CreatedAt: " + createdAt + ',' +
|
"CreatedAt: " + createdAt + ',' +
|
||||||
"StartIndexPtr: " + startIndexPtr + ',' +
|
"StartIndexPtr: " + startIndexPtr + ',' +
|
||||||
"EndIndexPtr: " + endIndexPtr +
|
"EndIndexPtr: " + endIndexPtr +
|
||||||
'}';
|
'}';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -9,22 +9,21 @@ package org.lionsoul.ip2region.xdb;
|
||||||
// @Date 2022/06/23
|
// @Date 2022/06/23
|
||||||
|
|
||||||
|
|
||||||
|
import java.io.Closeable;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.RandomAccessFile;
|
import java.io.RandomAccessFile;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
|
||||||
public class Searcher {
|
public class Searcher implements Closeable {
|
||||||
// constant defined copied from the xdb maker
|
// constant defined copied from the xdb maker
|
||||||
public static final int HeaderInfoLength = 256;
|
public static final int HeaderInfoLength = 256;
|
||||||
public static final int VectorIndexRows = 256;
|
public static final int VectorIndexRows = 256;
|
||||||
public static final int VectorIndexCols = 256;
|
public static final int VectorIndexCols = 256;
|
||||||
public static final int VectorIndexSize = 8;
|
public static final int VectorIndexSize = 8;
|
||||||
public static final int SegmentIndexSize = 14;
|
public static final int SegmentIndexSize = 14;
|
||||||
|
public static final byte[] shiftIndex = {24, 16, 8, 0};
|
||||||
// random access file handle for file based search
|
// random access file handle for file based search
|
||||||
private final RandomAccessFile handle;
|
private final RandomAccessFile handle;
|
||||||
|
|
||||||
private int ioCount = 0;
|
|
||||||
|
|
||||||
// vector index.
|
// vector index.
|
||||||
// use the byte[] instead of VectorIndex entry array to keep
|
// use the byte[] instead of VectorIndex entry array to keep
|
||||||
// the minimal memory allocation.
|
// the minimal memory allocation.
|
||||||
|
|
@ -34,20 +33,7 @@ public class Searcher {
|
||||||
private final byte[] contentBuff;
|
private final byte[] contentBuff;
|
||||||
|
|
||||||
// --- static method to create searchers
|
// --- static method to create searchers
|
||||||
|
private int ioCount = 0;
|
||||||
public static Searcher newWithFileOnly(String dbPath) throws IOException {
|
|
||||||
return new Searcher(dbPath, null, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Searcher newWithVectorIndex(String dbPath, byte[] vectorIndex) throws IOException {
|
|
||||||
return new Searcher(dbPath, vectorIndex, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Searcher newWithBuffer(byte[] cBuff) throws IOException {
|
|
||||||
return new Searcher(null, null, cBuff);
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- End of creator
|
|
||||||
|
|
||||||
public Searcher(String dbFile, byte[] vectorIndex, byte[] cBuff) throws IOException {
|
public Searcher(String dbFile, byte[] vectorIndex, byte[] cBuff) throws IOException {
|
||||||
if (cBuff != null) {
|
if (cBuff != null) {
|
||||||
|
|
@ -61,6 +47,130 @@ public class Searcher {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static Searcher newWithFileOnly(String dbPath) throws IOException {
|
||||||
|
return new Searcher(dbPath, null, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- End of creator
|
||||||
|
|
||||||
|
public static Searcher newWithVectorIndex(String dbPath, byte[] vectorIndex) throws IOException {
|
||||||
|
return new Searcher(dbPath, vectorIndex, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Searcher newWithBuffer(byte[] cBuff) throws IOException {
|
||||||
|
return new Searcher(null, null, cBuff);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Header loadHeader(RandomAccessFile handle) throws IOException {
|
||||||
|
handle.seek(0);
|
||||||
|
final byte[] buff = new byte[HeaderInfoLength];
|
||||||
|
handle.read(buff);
|
||||||
|
return new Header(buff);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Header loadHeaderFromFile(String dbPath) throws IOException {
|
||||||
|
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
|
||||||
|
final Header header = loadHeader(handle);
|
||||||
|
handle.close();
|
||||||
|
return header;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException {
|
||||||
|
handle.seek(HeaderInfoLength);
|
||||||
|
int len = VectorIndexRows * VectorIndexCols * VectorIndexSize;
|
||||||
|
final byte[] buff = new byte[len];
|
||||||
|
int rLen = handle.read(buff);
|
||||||
|
if (rLen != len) {
|
||||||
|
throw new IOException("incomplete read: read bytes should be " + len);
|
||||||
|
}
|
||||||
|
|
||||||
|
return buff;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] loadVectorIndexFromFile(String dbPath) throws IOException {
|
||||||
|
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
|
||||||
|
final byte[] vIndex = loadVectorIndex(handle);
|
||||||
|
handle.close();
|
||||||
|
return vIndex;
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- static cache util function
|
||||||
|
|
||||||
|
public static byte[] loadContent(RandomAccessFile handle) throws IOException {
|
||||||
|
handle.seek(0);
|
||||||
|
final byte[] buff = new byte[(int) handle.length()];
|
||||||
|
int rLen = handle.read(buff);
|
||||||
|
if (rLen != buff.length) {
|
||||||
|
throw new IOException("incomplete read: read bytes should be " + buff.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
return buff;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] loadContentFromFile(String dbPath) throws IOException {
|
||||||
|
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
|
||||||
|
final byte[] content = loadContent(handle);
|
||||||
|
handle.close();
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* get an int from a byte array start from the specified offset */
|
||||||
|
public static long getIntLong(byte[] b, int offset) {
|
||||||
|
return (
|
||||||
|
((b[offset++] & 0x000000FFL)) |
|
||||||
|
((b[offset++] << 8) & 0x0000FF00L) |
|
||||||
|
((b[offset++] << 16) & 0x00FF0000L) |
|
||||||
|
((b[offset] << 24) & 0xFF000000L)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static int getInt(byte[] b, int offset) {
|
||||||
|
return (
|
||||||
|
((b[offset++] & 0x000000FF)) |
|
||||||
|
((b[offset++] << 8) & 0x0000FF00) |
|
||||||
|
((b[offset++] << 16) & 0x00FF0000) |
|
||||||
|
((b[offset] << 24) & 0xFF000000)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static int getInt2(byte[] b, int offset) {
|
||||||
|
return (
|
||||||
|
((b[offset++] & 0x000000FF)) |
|
||||||
|
((b[offset] << 8) & 0x0000FF00)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* long int to ip string */
|
||||||
|
public static String long2ip(long ip) {
|
||||||
|
return String.valueOf((ip >> 24) & 0xFF) + '.' +
|
||||||
|
((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF);
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- End cache load util function
|
||||||
|
|
||||||
|
// --- static util method
|
||||||
|
|
||||||
|
/* check the specified ip address */
|
||||||
|
public static long checkIP(String ip) throws Exception {
|
||||||
|
String[] ps = ip.split("\\.");
|
||||||
|
if (ps.length != 4) {
|
||||||
|
throw new Exception("invalid ip address `" + ip + "`");
|
||||||
|
}
|
||||||
|
|
||||||
|
long ipDst = 0;
|
||||||
|
for (int i = 0; i < ps.length; i++) {
|
||||||
|
int val = Integer.parseInt(ps[i]);
|
||||||
|
if (val > 255) {
|
||||||
|
throw new Exception("ip part `" + ps[i] + "` should be less then 256");
|
||||||
|
}
|
||||||
|
|
||||||
|
ipDst |= ((long) val << shiftIndex[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
return ipDst & 0xFFFFFFFFL;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
public void close() throws IOException {
|
public void close() throws IOException {
|
||||||
if (this.handle != null) {
|
if (this.handle != null) {
|
||||||
this.handle.close();
|
this.handle.close();
|
||||||
|
|
@ -135,7 +245,7 @@ public class Searcher {
|
||||||
// load and return the region data
|
// load and return the region data
|
||||||
final byte[] regionBuff = new byte[dataLen];
|
final byte[] regionBuff = new byte[dataLen];
|
||||||
read(dataPtr, regionBuff);
|
read(dataPtr, regionBuff);
|
||||||
return new String(regionBuff, "utf-8");
|
return new String(regionBuff, StandardCharsets.UTF_8);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void read(int offset, byte[] buffer) throws IOException {
|
protected void read(int offset, byte[] buffer) throws IOException {
|
||||||
|
|
@ -157,116 +267,4 @@ public class Searcher {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- static cache util function
|
}
|
||||||
|
|
||||||
public static Header loadHeader(RandomAccessFile handle) throws IOException {
|
|
||||||
handle.seek(0);
|
|
||||||
final byte[] buff = new byte[HeaderInfoLength];
|
|
||||||
handle.read(buff);
|
|
||||||
return new Header(buff);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Header loadHeaderFromFile(String dbPath) throws IOException {
|
|
||||||
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
|
|
||||||
final Header header = loadHeader(handle);
|
|
||||||
handle.close();
|
|
||||||
return header;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException {
|
|
||||||
handle.seek(HeaderInfoLength);
|
|
||||||
int len = VectorIndexRows * VectorIndexCols * VectorIndexSize;
|
|
||||||
final byte[] buff = new byte[len];
|
|
||||||
int rLen = handle.read(buff);
|
|
||||||
if (rLen != len) {
|
|
||||||
throw new IOException("incomplete read: read bytes should be " + len);
|
|
||||||
}
|
|
||||||
|
|
||||||
return buff;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static byte[] loadVectorIndexFromFile(String dbPath) throws IOException {
|
|
||||||
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
|
|
||||||
final byte[] vIndex = loadVectorIndex(handle);
|
|
||||||
handle.close();
|
|
||||||
return vIndex;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static byte[] loadContent(RandomAccessFile handle) throws IOException {
|
|
||||||
handle.seek(0);
|
|
||||||
final byte[] buff = new byte[(int) handle.length()];
|
|
||||||
int rLen = handle.read(buff);
|
|
||||||
if (rLen != buff.length) {
|
|
||||||
throw new IOException("incomplete read: read bytes should be " + buff.length);
|
|
||||||
}
|
|
||||||
|
|
||||||
return buff;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static byte[] loadContentFromFile(String dbPath) throws IOException {
|
|
||||||
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
|
|
||||||
final byte[] content = loadContent(handle);
|
|
||||||
handle.close();
|
|
||||||
return content;
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- End cache load util function
|
|
||||||
|
|
||||||
// --- static util method
|
|
||||||
|
|
||||||
/* get an int from a byte array start from the specified offset */
|
|
||||||
public static long getIntLong(byte[] b, int offset) {
|
|
||||||
return (
|
|
||||||
((b[offset++] & 0x000000FFL)) |
|
|
||||||
((b[offset++] << 8) & 0x0000FF00L) |
|
|
||||||
((b[offset++] << 16) & 0x00FF0000L) |
|
|
||||||
((b[offset ] << 24) & 0xFF000000L)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static int getInt(byte[] b, int offset) {
|
|
||||||
return (
|
|
||||||
((b[offset++] & 0x000000FF)) |
|
|
||||||
((b[offset++] << 8) & 0x0000FF00) |
|
|
||||||
((b[offset++] << 16) & 0x00FF0000) |
|
|
||||||
((b[offset ] << 24) & 0xFF000000)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static int getInt2(byte[] b, int offset) {
|
|
||||||
return (
|
|
||||||
((b[offset++] & 0x000000FF)) |
|
|
||||||
((b[offset ] << 8) & 0x0000FF00)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* long int to ip string */
|
|
||||||
public static String long2ip( long ip )
|
|
||||||
{
|
|
||||||
return String.valueOf((ip >> 24) & 0xFF) + '.' +
|
|
||||||
((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static final byte[] shiftIndex = {24, 16, 8, 0};
|
|
||||||
|
|
||||||
/* check the specified ip address */
|
|
||||||
public static long checkIP(String ip) throws Exception {
|
|
||||||
String[] ps = ip.split("\\.");
|
|
||||||
if (ps.length != 4) {
|
|
||||||
throw new Exception("invalid ip address `" + ip + "`");
|
|
||||||
}
|
|
||||||
|
|
||||||
long ipDst = 0;
|
|
||||||
for (int i = 0; i < ps.length; i++) {
|
|
||||||
int val = Integer.parseInt(ps[i]);
|
|
||||||
if (val > 255) {
|
|
||||||
throw new Exception("ip part `"+ps[i]+"` should be less then 256");
|
|
||||||
}
|
|
||||||
|
|
||||||
ipDst |= ((long) val << shiftIndex[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
return ipDst & 0xFFFFFFFFL;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue