xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge 83f69936b8 into 3ef87ff727

This commit is contained in:
Ann Velazquez 2023-10-11 13:18:00 -06:00 committed by GitHub
parent 3ef87ff727 83f69936b8
commit cb96476f48
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 149 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
binding/java/ReadMe.md
View File

@ -15,34 +15,28 @@
```java
import org.lionsoul.ip2region.xdb.Searcher;
import java.io.*;
import java.util.concurrent.TimeUnit;
public class SearcherTest {
public static void main(String[] args) {
// 1、创建 searcher 对象
String dbPath = "ip2region.xdb file path";
Searcher searcher = null;
try {
searcher = Searcher.newWithFileOnly(dbPath);
} catch (IOException e) {
System.out.printf("failed to create searcher with `%s`: %s\n", dbPath, e);
return;
}
// 2、查询
try {
String ip = "1.2.3.4";
// 1、创建 searcher 对象
try (Searcher searcher = Searcher.newWithFileOnly(dbPath)) {
// 2、查询
long sTime = System.nanoTime();
String region = searcher.search(ip);
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime);
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
} catch (IOException e) {
System.out.printf("failed to create searcher with `%s`: %s\n", dbPath, e);
} catch (Exception e) {
System.out.printf("failed to search(%s): %s\n", ip, e);
}
// 3、关闭资源
searcher.close();
// 3、关闭资源（这里通过 try-with-resources 自动关闭）
// 备注：并发使用，每个线程需要创建一个独立的 searcher 对象单独使用。
}
@ -54,12 +48,14 @@ public class SearcherTest {
我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据，然后全局缓存，每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作，从而加速查询，减少 IO 压力。
```java
import org.lionsoul.ip2region.xdb.Searcher;
import java.io.*;
import java.util.concurrent.TimeUnit;
public class SearcherTest {
public static void main(String[] args) {
String dbPath = "ip2region.xdb file path";
String ip = "1.2.3.4";
// 1、从 dbPath 中预先加载 VectorIndex 缓存，并且把这个得到的数据作为全局变量，后续反复使用。
byte[] vIndex;
@ -71,27 +67,17 @@ public class SearcherTest {
}
// 2、使用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。
Searcher searcher;
try {
searcher = Searcher.newWithVectorIndex(dbPath, vIndex);
} catch (Exception e) {
System.out.printf("failed to create vectorIndex cached searcher with `%s`: %s\n", dbPath, e);
return;
}
// 3、查询
try {
String ip = "1.2.3.4";
try (Searcher searcher = Searcher.newWithVectorIndex(dbPath, vIndex)) {
long sTime = System.nanoTime();
String region = searcher.search(ip);
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime);
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
} catch (IOException e) {
System.out.printf("failed to create vectorIndex cached searcher with `%s`: %s\n", dbPath, e);
} catch (Exception e) {
System.out.printf("failed to search(%s): %s\n", ip, e);
}
// 4、关闭资源
searcher.close();
// 3、关闭资源（这里通过 try-with-resources 自动关闭）
// 备注：每个线程需要单独创建一个独立的 Searcher 对象，但是都共享全局的制度 vIndex 缓存。
}
@ -103,12 +89,14 @@ public class SearcherTest {
我们也可以预先加载整个 ip2region.xdb 的数据到内存，然后基于这个数据创建查询对象来实现完全基于文件的查询，类似之前的 memory search。
```java
import org.lionsoul.ip2region.xdb.Searcher;
import java.io.*;
import java.util.concurrent.TimeUnit;
public class SearcherTest {
public static void main(String[] args) {
String dbPath = "ip2region.xdb file path";
String ip = "1.2.3.4";
// 1、从 dbPath 加载整个 xdb 到内存。
byte[] cBuff;
@ -123,17 +111,16 @@ public class SearcherTest {
Searcher searcher;
try {
searcher = Searcher.newWithBuffer(cBuff);
} catch (Exception e) {
} catch (IOException e) {
System.out.printf("failed to create content cached searcher: %s\n", e);
return;
}
// 3、查询
try {
String ip = "1.2.3.4";
long sTime = System.nanoTime();
String region = searcher.search(ip);
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime);
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
} catch (Exception e) {
System.out.printf("failed to search(%s): %s\n", ip, e);

10
binding/java/pom.xml
View File

@ -41,6 +41,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
<maven.compiler.source>1.6</maven.compiler.source>
<maven.compiler.target>1.6</maven.compiler.target>
</properties>
@ -112,6 +113,15 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>
</plugins>
</build>

39
binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java
View File

@ -8,7 +8,10 @@ package org.lionsoul.ip2region;
import org.lionsoul.ip2region.xdb.Searcher;
import java.io.*;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.concurrent.TimeUnit;
public class SearchTest {
@ -65,7 +68,7 @@ public class SearchTest {
}
}
if (dbPath.length() < 1) {
if (dbPath.isEmpty()) {
System.out.print("java -jar ip2region-{version}.jar search [command options]\n");
System.out.print("options:\n");
System.out.print(" --db string ip2region binary xdb file path\n");
@ -73,8 +76,8 @@ public class SearchTest {
return;
}
Searcher searcher = createSearcher(dbPath, cachePolicy);
final BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
try (Searcher searcher = createSearcher(dbPath, cachePolicy);
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in))) {
System.out.printf("ip2region xdb searcher test program, cachePolicy: %s\ntype 'quit' to exit\n", cachePolicy);
while (true) {
System.out.print("ip2region>> ");
@ -83,7 +86,7 @@ public class SearchTest {
continue;
}
if ( line.equalsIgnoreCase("quit") ) {
if ("quit".equalsIgnoreCase(line)) {
break;
}
@ -96,9 +99,7 @@ public class SearchTest {
System.out.printf("{err: %s, ioCount: %d}\n", e, searcher.getIOCount());
}
}
reader.close();
searcher.close();
}
System.out.println("searcher test program exited, thanks for trying");
}
@ -121,19 +122,23 @@ public class SearchTest {
String key = r.substring(2, sIdx);
String val = r.substring(sIdx + 1);
if ("db".equals(key)) {
switch (key) {
case "db":
dbPath = val;
} else if ("src".equals(key)) {
break;
case "src":
srcPath = val;
} else if ("cache-policy".equals(key)) {
break;
case "cache-policy":
cachePolicy = val;
} else {
break;
default:
System.out.printf("undefined option `%s`\n", r);
return;
}
}
if (dbPath.length() < 1 || srcPath.length() < 1) {
if (dbPath.isEmpty() || srcPath.isEmpty()) {
System.out.print("java -jar ip2region-{version}.jar bench [command options]\n");
System.out.print("options:\n");
System.out.print(" --db string ip2region binary xdb file path\n");
@ -142,10 +147,10 @@ public class SearchTest {
return;
}
Searcher searcher = createSearcher(dbPath, cachePolicy);
long count = 0, costs = 0, tStart = System.nanoTime();
try (Searcher searcher = createSearcher(dbPath, cachePolicy);
BufferedReader reader = new BufferedReader(new FileReader(srcPath))) {
String line;
final BufferedReader reader = new BufferedReader(new FileReader(srcPath));
while ((line = reader.readLine()) != null) {
String l = line.trim();
String[] ps = l.split("\\|", 3);
@ -190,9 +195,7 @@ public class SearchTest {
count++;
}
}
reader.close();
searcher.close();
}
long took = System.nanoTime() - tStart;
System.out.printf("Bench finished, {cachePolicy: %s, total: %d, took: %ds, cost: %d μs/op}\n",
cachePolicy, count, TimeUnit.NANOSECONDS.toSeconds(took),

2
binding/java/src/main/java/org/lionsoul/ip2region/UtilTest.java
View File

@ -12,7 +12,7 @@ public class UtilTest {
public static void testIP2Long() {
String ip = "1.2.3.4";
long ipAddr = 0;
long ipAddr;
try {
ipAddr = Searcher.checkIP(ip);
} catch (Exception e) {

3
binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java
View File

@ -24,7 +24,8 @@ public class Header {
buffer = buff;
}
@Override public String toString() {
@Override
public String toString() {
return "{" +
"Version: " + version + ',' +
"IndexPolicy: " + indexPolicy + ',' +

28
binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java
View File

@ -9,10 +9,12 @@ package org.lionsoul.ip2region.xdb;
// @Date 2022/06/23
import java.io.Closeable;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.charset.StandardCharsets;
public class Searcher {
public class Searcher implements Closeable {
// constant defined copied from the xdb maker
public static final int HeaderInfoLength = 256;
public static final int VectorIndexRows = 256;
@ -61,6 +63,7 @@ public class Searcher {
}
}
@Override
public void close() throws IOException {
if (this.handle != null) {
this.handle.close();
@ -135,7 +138,7 @@ public class Searcher {
// load and return the region data
final byte[] regionBuff = new byte[dataLen];
read(dataPtr, regionBuff);
return new String(regionBuff, "utf-8");
return new String(regionBuff, StandardCharsets.UTF_8);
}
protected void read(int offset, byte[] buffer) throws IOException {
@ -167,10 +170,9 @@ public class Searcher {
}
public static Header loadHeaderFromFile(String dbPath) throws IOException {
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
final Header header = loadHeader(handle);
handle.close();
return header;
try (RandomAccessFile handle = new RandomAccessFile(dbPath, "r")) {
return loadHeader(handle);
}
}
public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException {
@ -186,10 +188,9 @@ public class Searcher {
}
public static byte[] loadVectorIndexFromFile(String dbPath) throws IOException {
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
final byte[] vIndex = loadVectorIndex(handle);
handle.close();
return vIndex;
try (RandomAccessFile handle = new RandomAccessFile(dbPath, "r")) {
return loadVectorIndex(handle);
}
}
public static byte[] loadContent(RandomAccessFile handle) throws IOException {
@ -204,10 +205,9 @@ public class Searcher {
}
public static byte[] loadContentFromFile(String dbPath) throws IOException {
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
final byte[] content = loadContent(handle);
handle.close();
return content;
try (RandomAccessFile handle = new RandomAccessFile(dbPath, "r")) {
return loadContent(handle);
}
}
// --- End cache load util function