Merge pull request #408 from lionsoul2014/fr_java_easier_to_build
Ip2Region Config build from xdb File Object or xdb InputStream
This commit is contained in:
commit
9669585216
|
|
@ -7,7 +7,7 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.lionsoul</groupId>
|
<groupId>org.lionsoul</groupId>
|
||||||
<artifactId>ip2region</artifactId>
|
<artifactId>ip2region</artifactId>
|
||||||
<version>3.3.0</version>
|
<version>3.3.1</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -21,6 +21,8 @@ import org.lionsoul.ip2region.service.Ip2Region;
|
||||||
final Config v4Config = Config.custom()
|
final Config v4Config = Config.custom()
|
||||||
.setCachePolicy(Config.VIndexCache) // 指定缓存策略: NoCache / VIndexCache / BufferCache
|
.setCachePolicy(Config.VIndexCache) // 指定缓存策略: NoCache / VIndexCache / BufferCache
|
||||||
.setSearchers(15) // 设置初始化的查询器数量
|
.setSearchers(15) // 设置初始化的查询器数量
|
||||||
|
// .setXdbInputStream(InputStream) // 设置 v4 xdb 文件的 inputstream 对象
|
||||||
|
// .setXdbFile(File) // 设置 v4 xdb File 对象
|
||||||
.setXdbPath("ip2region v4 xdb path") // 设置 v4 xdb 文件的路径
|
.setXdbPath("ip2region v4 xdb path") // 设置 v4 xdb 文件的路径
|
||||||
.asV4(); // 指定为 v4 配置
|
.asV4(); // 指定为 v4 配置
|
||||||
|
|
||||||
|
|
@ -28,9 +30,13 @@ final Config v4Config = Config.custom()
|
||||||
final Config v6Config = Config.custom()
|
final Config v6Config = Config.custom()
|
||||||
.setCachePolicy(Config.VIndexCache) // 指定缓存策略: NoCache / VIndexCache / BufferCache
|
.setCachePolicy(Config.VIndexCache) // 指定缓存策略: NoCache / VIndexCache / BufferCache
|
||||||
.setSearchers(15) // 设置初始化的查询器数量
|
.setSearchers(15) // 设置初始化的查询器数量
|
||||||
|
// .setXdbInputStream(InputStream) // 设置 v6 xdb 文件的 inputstream 对象
|
||||||
|
// .setXdbFile(File) // 设置 v6 xdb File 对象
|
||||||
.setXdbPath("ip2region v6 xdb path") // 设置 v6 xdb 文件的路径
|
.setXdbPath("ip2region v6 xdb path") // 设置 v6 xdb 文件的路径
|
||||||
.asV6(); // 指定为 v6 配置
|
.asV6(); // 指定为 v6 配置
|
||||||
|
|
||||||
|
// 备注:Xdb 三种初始化输入的优先级:XdbInputStream -> XdbFile -> XdbPath
|
||||||
|
|
||||||
// 3,通过上述配置创建 Ip2Region 查询服务
|
// 3,通过上述配置创建 Ip2Region 查询服务
|
||||||
final Ip2Region ip2Region = Ip2Region.create(v4Config, v6Config);
|
final Ip2Region ip2Region = Ip2Region.create(v4Config, v6Config);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@
|
||||||
|
|
||||||
<groupId>org.lionsoul</groupId>
|
<groupId>org.lionsoul</groupId>
|
||||||
<artifactId>ip2region</artifactId>
|
<artifactId>ip2region</artifactId>
|
||||||
<version>3.3.0</version>
|
<version>3.3.1</version>
|
||||||
<packaging>jar</packaging>
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
<name>ip2region</name>
|
<name>ip2region</name>
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@
|
||||||
package org.lionsoul.ip2region;
|
package org.lionsoul.ip2region;
|
||||||
|
|
||||||
import org.lionsoul.ip2region.service.Config;
|
import org.lionsoul.ip2region.service.Config;
|
||||||
import org.lionsoul.ip2region.service.InvalidCachePolicyException;
|
import org.lionsoul.ip2region.service.InvalidConfigException;
|
||||||
import org.lionsoul.ip2region.service.Ip2Region;
|
import org.lionsoul.ip2region.service.Ip2Region;
|
||||||
import org.lionsoul.ip2region.xdb.InetAddressException;
|
import org.lionsoul.ip2region.xdb.InetAddressException;
|
||||||
import org.lionsoul.ip2region.xdb.XdbException;
|
import org.lionsoul.ip2region.xdb.XdbException;
|
||||||
|
|
@ -49,7 +49,7 @@ public class SearcherTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static final Ip2Region createService(
|
public static final Ip2Region createService(
|
||||||
String v4XdbPath, String v4CachePolicy, String v6XdbPath, String v6CachePolicy) throws IOException, XdbException, InvalidCachePolicyException {
|
String v4XdbPath, String v4CachePolicy, String v6XdbPath, String v6CachePolicy) throws IOException, XdbException, InvalidConfigException {
|
||||||
final Config v4Config = Config.custom()
|
final Config v4Config = Config.custom()
|
||||||
.setCachePolicy(Config.cachePolicyFromName(v4CachePolicy))
|
.setCachePolicy(Config.cachePolicyFromName(v4CachePolicy))
|
||||||
.setSearchers(1)
|
.setSearchers(1)
|
||||||
|
|
@ -89,7 +89,7 @@ public class SearcherTest {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void searchTest(String[] args) throws IOException, XdbException, InvalidCachePolicyException, InterruptedException {
|
public static void searchTest(String[] args) throws IOException, XdbException, InterruptedException, InvalidConfigException {
|
||||||
String help = "";
|
String help = "";
|
||||||
String v4DbPath = "", v4CachePolicy = "vectorIndex";
|
String v4DbPath = "", v4CachePolicy = "vectorIndex";
|
||||||
String v6DbPath = "", v6CachePolicy = "vectorIndex";
|
String v6DbPath = "", v6CachePolicy = "vectorIndex";
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@
|
||||||
|
|
||||||
package org.lionsoul.ip2region.service;
|
package org.lionsoul.ip2region.service;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
import org.lionsoul.ip2region.xdb.Header;
|
import org.lionsoul.ip2region.xdb.Header;
|
||||||
|
|
@ -22,12 +23,15 @@ public class Config {
|
||||||
public static final int VIndexCache = 1;
|
public static final int VIndexCache = 1;
|
||||||
public static final int BufferCache = 2;
|
public static final int BufferCache = 2;
|
||||||
|
|
||||||
|
// alias of BufferCache but easier to understand or Remember
|
||||||
|
public static final int FullCache = 2;
|
||||||
|
|
||||||
// search cache policy
|
// search cache policy
|
||||||
public final int cachePolicy;
|
public final int cachePolicy;
|
||||||
public final Version ipVersion;
|
public final Version ipVersion;
|
||||||
|
|
||||||
// xdb file path
|
// xdb file path
|
||||||
public final String xdbPath;
|
public final File xdbFile;
|
||||||
public final Header header;
|
public final Header header;
|
||||||
|
|
||||||
public final byte[] vIndex;
|
public final byte[] vIndex;
|
||||||
|
|
@ -40,21 +44,21 @@ public class Config {
|
||||||
return new ConfigBuilder();
|
return new ConfigBuilder();
|
||||||
}
|
}
|
||||||
|
|
||||||
protected Config(int cachePolicy, Version ipVersion, String xdbPath,
|
protected Config(int cachePolicy, Version ipVersion, File xdbFile,
|
||||||
Header header, byte[] vIndex, LongByteArray cBuffer, int searchers) throws IOException, XdbException {
|
Header header, byte[] vIndex, LongByteArray cBuffer, int searchers) throws IOException, XdbException {
|
||||||
this.cachePolicy = cachePolicy;
|
this.cachePolicy = cachePolicy;
|
||||||
this.ipVersion = ipVersion;
|
this.ipVersion = ipVersion;
|
||||||
|
|
||||||
this.xdbPath = xdbPath;
|
this.xdbFile = xdbFile;
|
||||||
this.header = header;
|
this.header = header;
|
||||||
this.vIndex = vIndex;
|
this.vIndex = vIndex;
|
||||||
this.cBuffer = cBuffer;
|
this.cBuffer = cBuffer;
|
||||||
|
|
||||||
final Version xVersion = Version.fromHeader(header);
|
final Version xVersion = Version.fromHeader(header);
|
||||||
// verify the ip version (ipVersion and the version of the xdb file should be the same)
|
// verify the ip version (ipVersion and the version of the xdb file should be the same)
|
||||||
if (header.ipVersion != ipVersion.id) {
|
if (header.ipVersion != ipVersion.id) {
|
||||||
throw new XdbException("ip verison not match: xdb file "
|
throw new XdbException("ip verison not match: xdb file "
|
||||||
+ xdbPath + " (" + xVersion.name + "), as " + ipVersion.name + " expected");
|
+ xdbFile.getAbsolutePath() + " (" + xVersion.name + "), as " + ipVersion.name + " expected");
|
||||||
}
|
}
|
||||||
|
|
||||||
this.searchers = searchers;
|
this.searchers = searchers;
|
||||||
|
|
@ -65,7 +69,7 @@ public class Config {
|
||||||
sb.append('{');
|
sb.append('{');
|
||||||
sb.append("cache_policy:").append(cachePolicy).append(',');
|
sb.append("cache_policy:").append(cachePolicy).append(',');
|
||||||
sb.append("version:").append(ipVersion.toString()).append(',');
|
sb.append("version:").append(ipVersion.toString()).append(',');
|
||||||
sb.append("xdb_path:").append(xdbPath).append(',');
|
sb.append("xdb_path:").append(xdbFile == null ? "null" : xdbFile.getAbsolutePath()).append(',');
|
||||||
sb.append("header:").append(header.toString()).append(',');
|
sb.append("header:").append(header.toString()).append(',');
|
||||||
if (vIndex == null) {
|
if (vIndex == null) {
|
||||||
sb.append("v_index: null, ");
|
sb.append("v_index: null, ");
|
||||||
|
|
@ -82,7 +86,7 @@ public class Config {
|
||||||
return sb.toString();
|
return sb.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static final int cachePolicyFromName(String name) throws InvalidCachePolicyException {
|
public static final int cachePolicyFromName(String name) throws InvalidConfigException {
|
||||||
final String lName = name.toLowerCase();
|
final String lName = name.toLowerCase();
|
||||||
if (lName.equals("file") || lName.equals("nocache")) {
|
if (lName.equals("file") || lName.equals("nocache")) {
|
||||||
return NoCache;
|
return NoCache;
|
||||||
|
|
@ -91,7 +95,7 @@ public class Config {
|
||||||
} else if (lName.equals("content") || lName.equals("buffercache")) {
|
} else if (lName.equals("content") || lName.equals("buffercache")) {
|
||||||
return BufferCache;
|
return BufferCache;
|
||||||
} else {
|
} else {
|
||||||
throw new InvalidCachePolicyException("invalid cache policy `" + name + "`");
|
throw new InvalidConfigException("invalid cache policy `" + name + "`");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -4,7 +4,9 @@
|
||||||
|
|
||||||
package org.lionsoul.ip2region.service;
|
package org.lionsoul.ip2region.service;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
import java.io.RandomAccessFile;
|
import java.io.RandomAccessFile;
|
||||||
|
|
||||||
import org.lionsoul.ip2region.xdb.Header;
|
import org.lionsoul.ip2region.xdb.Header;
|
||||||
|
|
@ -23,8 +25,11 @@ public class ConfigBuilder {
|
||||||
// cache policy
|
// cache policy
|
||||||
private int cachePolicy = Config.VIndexCache;
|
private int cachePolicy = Config.VIndexCache;
|
||||||
|
|
||||||
// xdb file path
|
// xdb file path / Object / InputStream.
|
||||||
|
// Priority: InputStream -> File Object -> String path
|
||||||
private String xdbPath = null;
|
private String xdbPath = null;
|
||||||
|
private File xdbFile = null;
|
||||||
|
private InputStream xdbInputStream = null;
|
||||||
|
|
||||||
// searchers
|
// searchers
|
||||||
private int searchers = 20;
|
private int searchers = 20;
|
||||||
|
|
@ -45,14 +50,53 @@ public class ConfigBuilder {
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public ConfigBuilder setXdbFile(File xdbFile) {
|
||||||
|
this.xdbFile = xdbFile;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ConfigBuilder setXdbInputStream(InputStream xdbInputStream) {
|
||||||
|
this.xdbInputStream = xdbInputStream;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
public ConfigBuilder setSearchers(int searchers) {
|
public ConfigBuilder setSearchers(int searchers) {
|
||||||
this.searchers = searchers;
|
this.searchers = searchers;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
private Config build(Version ipVersion) throws IOException, XdbException {
|
private Config build(Version ipVersion) throws IOException, XdbException, InvalidConfigException {
|
||||||
|
if (xdbInputStream == null) {
|
||||||
|
// everyting is fine
|
||||||
|
} else if (cachePolicy != Config.BufferCache) {
|
||||||
|
// @Note: we can't directly rewrite the cachePolicy to Config.BufferCache.
|
||||||
|
// you must know what you are doing.
|
||||||
|
throw new InvalidConfigException("SetXdbInputStream could ONLY be used with cachePolicy = Config.BufferCache");
|
||||||
|
} else {
|
||||||
|
// 1, load the content buffer
|
||||||
|
final LongByteArray cBuffer = Searcher.loadContentFromInputStream(xdbInputStream);
|
||||||
|
|
||||||
|
// 2, verify the xdb from the buffer
|
||||||
|
Searcher.verify(Searcher.loadHeaderFromBuffer(cBuffer), cBuffer.length());
|
||||||
|
|
||||||
|
// 3, load the header
|
||||||
|
final Header header = Searcher.loadHeaderFromBuffer(cBuffer);
|
||||||
|
|
||||||
|
// create the config without xdbFile and vIndex
|
||||||
|
return new Config(cachePolicy, ipVersion, null, header, null, cBuffer, searchers);
|
||||||
|
}
|
||||||
|
|
||||||
// load the header and the cache buffer
|
// load the header and the cache buffer
|
||||||
final RandomAccessFile raf = new RandomAccessFile(xdbPath, "r");
|
final File xdbFile;
|
||||||
|
if (this.xdbFile != null) {
|
||||||
|
xdbFile = this.xdbFile;
|
||||||
|
} else if (this.xdbPath != null) {
|
||||||
|
xdbFile = new File(this.xdbPath);
|
||||||
|
} else {
|
||||||
|
throw new InvalidConfigException("Both xdbFile and xdbPath is null");
|
||||||
|
}
|
||||||
|
|
||||||
|
final RandomAccessFile raf = new RandomAccessFile(xdbFile, "r");
|
||||||
|
|
||||||
// 1, verify the xdb
|
// 1, verify the xdb
|
||||||
Searcher.verify(raf);
|
Searcher.verify(raf);
|
||||||
|
|
@ -67,16 +111,16 @@ public class ConfigBuilder {
|
||||||
final LongByteArray cBuffer = cachePolicy == Config.BufferCache ? Searcher.loadContent(raf) : null;
|
final LongByteArray cBuffer = cachePolicy == Config.BufferCache ? Searcher.loadContent(raf) : null;
|
||||||
|
|
||||||
raf.close();
|
raf.close();
|
||||||
return new Config(cachePolicy, ipVersion, xdbPath, header, vIndex, cBuffer, searchers);
|
return new Config(cachePolicy, ipVersion, xdbFile, header, vIndex, cBuffer, searchers);
|
||||||
}
|
}
|
||||||
|
|
||||||
// build the final #Config instance for IPv4
|
// build the final #Config instance for IPv4
|
||||||
public Config asV4() throws IOException, XdbException {
|
public Config asV4() throws IOException, XdbException, InvalidConfigException {
|
||||||
return build(Version.IPv4);
|
return build(Version.IPv4);
|
||||||
}
|
}
|
||||||
|
|
||||||
// build the final #Config instance for IPv6
|
// build the final #Config instance for IPv6
|
||||||
public Config asV6() throws IOException, XdbException {
|
public Config asV6() throws IOException, XdbException, InvalidConfigException {
|
||||||
return build(Version.IPv6);
|
return build(Version.IPv6);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
// Copyright 2022 The Ip2Region Authors. All rights reserved.
|
|
||||||
// Use of this source code is governed by a Apache2.0-style
|
|
||||||
// license that can be found in the LICENSE file.
|
|
||||||
|
|
||||||
package org.lionsoul.ip2region.service;
|
|
||||||
|
|
||||||
public class InvalidCachePolicyException extends Exception {
|
|
||||||
public InvalidCachePolicyException(String str) {
|
|
||||||
super(str);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
package org.lionsoul.ip2region.service;
|
||||||
|
|
||||||
|
public class InvalidConfigException extends Exception {
|
||||||
|
public InvalidConfigException(String str) {
|
||||||
|
super(str);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -4,6 +4,7 @@
|
||||||
|
|
||||||
package org.lionsoul.ip2region.service;
|
package org.lionsoul.ip2region.service;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
import org.lionsoul.ip2region.xdb.InetAddressException;
|
import org.lionsoul.ip2region.xdb.InetAddressException;
|
||||||
|
|
@ -37,23 +38,28 @@ public class Ip2Region {
|
||||||
return new Ip2Region(v4Config, v6Config).init();
|
return new Ip2Region(v4Config, v6Config).init();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static final Ip2Region create(final String v4XdbPath, final String v6XdbPath) throws IOException, XdbException {
|
public static final Ip2Region create(final String v4XdbPath, final String v6XdbPath) throws IOException, XdbException, InvalidConfigException {
|
||||||
return new Ip2Region(v4XdbPath, v6XdbPath).init();
|
return new Ip2Region(new File(v4XdbPath), new File(v6XdbPath)).init();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static final Ip2Region create(final File v4XdbFile, final File v6XdbFile) throws IOException, XdbException, InvalidConfigException {
|
||||||
|
return new Ip2Region(v4XdbFile, v6XdbFile).init();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* init the ip2reigon with two xdb file path and default cachePolicy vIndex.
|
* init the ip2reigon with two xdb file path and default cachePolicy vIndex.
|
||||||
* set it to null to disabled the search for specified version
|
* set it to null to disabled the search for specified version
|
||||||
*
|
*
|
||||||
* @param v4XdbPath
|
* @param v4XdbFile
|
||||||
* @param v6XdbPath
|
* @param v6XdbFile
|
||||||
* @throws XdbException
|
* @throws XdbException
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
|
* @throws InvalidConfigException
|
||||||
*/
|
*/
|
||||||
protected Ip2Region(String v4XdbPath, String v6XdbPath) throws IOException, XdbException {
|
protected Ip2Region(File v4XdbFile, File v6XdbFile) throws IOException, XdbException, InvalidConfigException {
|
||||||
this(
|
this(
|
||||||
v4XdbPath == null ? null : Config.custom().setXdbPath(v4XdbPath).asV4(),
|
v4XdbFile == null ? null : Config.custom().setXdbFile(v4XdbFile).asV4(),
|
||||||
v6XdbPath == null ? null : Config.custom().setXdbPath(v6XdbPath).asV6()
|
v6XdbFile == null ? null : Config.custom().setXdbFile(v6XdbFile).asV6()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -60,7 +60,7 @@ public class SearcherPool {
|
||||||
protected SearcherPool init() throws IOException {
|
protected SearcherPool init() throws IOException {
|
||||||
// create the searchers
|
// create the searchers
|
||||||
for (int i = pool.size(); i < config.searchers; i++) {
|
for (int i = pool.size(); i < config.searchers; i++) {
|
||||||
final Searcher searcher = new Searcher(config.ipVersion, config.xdbPath, config.vIndex, config.cBuffer);
|
final Searcher searcher = new Searcher(config.ipVersion, config.xdbFile, config.vIndex, config.cBuffer);
|
||||||
pool.add(searcher);
|
pool.add(searcher);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -33,13 +33,13 @@ public class Header {
|
||||||
|
|
||||||
@Override public String toString() {
|
@Override public String toString() {
|
||||||
return "{" +
|
return "{" +
|
||||||
"Version: " + version + ',' +
|
"Version:" + version + ',' +
|
||||||
"IndexPolicy: " + indexPolicy + ',' +
|
"IndexPolicy:" + indexPolicy + ',' +
|
||||||
"CreatedAt: " + createdAt + ',' +
|
"CreatedAt:" + createdAt + ',' +
|
||||||
"StartIndexPtr: " + startIndexPtr + ',' +
|
"StartIndexPtr:" + startIndexPtr + ',' +
|
||||||
"EndIndexPtr: " + endIndexPtr + ',' +
|
"EndIndexPtr:" + endIndexPtr + ',' +
|
||||||
"IPVersion: " + ipVersion + ',' +
|
"IPVersion:" + ipVersion + ',' +
|
||||||
"RuntimePtrBytes: " + runtimePtrBytes +
|
"RuntimePtrBytes:" + runtimePtrBytes +
|
||||||
'}';
|
'}';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,11 +4,14 @@
|
||||||
|
|
||||||
package org.lionsoul.ip2region.xdb;
|
package org.lionsoul.ip2region.xdb;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
|
||||||
// xdb searcher (Not thread safe implementation)
|
// xdb searcher (Not thread safe implementation)
|
||||||
// @Author Lion <chenxin619315@gmail.com>
|
// @Author Lion <chenxin619315@gmail.com>
|
||||||
// @Date 2022/06/23
|
// @Date 2022/06/23
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
import java.io.RandomAccessFile;
|
import java.io.RandomAccessFile;
|
||||||
|
|
||||||
public class Searcher {
|
public class Searcher {
|
||||||
|
|
@ -29,7 +32,7 @@ public class Searcher {
|
||||||
private final Version version;
|
private final Version version;
|
||||||
|
|
||||||
// random access file handle for file-based search
|
// random access file handle for file-based search
|
||||||
private final String xdbPath;
|
private final File xdbFile;
|
||||||
private final RandomAccessFile handle;
|
private final RandomAccessFile handle;
|
||||||
|
|
||||||
private int ioCount = 0;
|
private int ioCount = 0;
|
||||||
|
|
@ -46,12 +49,20 @@ public class Searcher {
|
||||||
|
|
||||||
// --- static method to create searchers
|
// --- static method to create searchers
|
||||||
|
|
||||||
public static Searcher newWithFileOnly(Version version, String dbPath) throws IOException {
|
public static Searcher newWithFileOnly(Version version, String xdbPath) throws IOException {
|
||||||
return new Searcher(version, dbPath, null, null);
|
return new Searcher(version, new File(xdbPath), null, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Searcher newWithVectorIndex(Version version, String dbPath, byte[] vectorIndex) throws IOException {
|
public static Searcher newWithFileOnly(Version version, File xdbFile) throws IOException {
|
||||||
return new Searcher(version, dbPath, vectorIndex, null);
|
return new Searcher(version, xdbFile, null, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Searcher newWithVectorIndex(Version version, String xdbPath, byte[] vectorIndex) throws IOException {
|
||||||
|
return new Searcher(version, new File(xdbPath), vectorIndex, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Searcher newWithVectorIndex(Version version, File xdbFile, byte[] vectorIndex) throws IOException {
|
||||||
|
return new Searcher(version, xdbFile, vectorIndex, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Searcher newWithBuffer(Version version, LongByteArray cBuff) throws IOException {
|
public static Searcher newWithBuffer(Version version, LongByteArray cBuff) throws IOException {
|
||||||
|
|
@ -60,15 +71,15 @@ public class Searcher {
|
||||||
|
|
||||||
// --- End of creator
|
// --- End of creator
|
||||||
|
|
||||||
public Searcher(Version version, String dbFile, byte[] vectorIndex, LongByteArray cBuff) throws IOException {
|
public Searcher(Version version, File xdbFile, byte[] vectorIndex, LongByteArray cBuff) throws IOException {
|
||||||
this.version = version;
|
this.version = version;
|
||||||
this.xdbPath = dbFile;
|
this.xdbFile = xdbFile;
|
||||||
if (cBuff != null) {
|
if (cBuff != null) {
|
||||||
this.handle = null;
|
this.handle = null;
|
||||||
this.vectorIndex = null;
|
this.vectorIndex = null;
|
||||||
this.contentBuff = cBuff;
|
this.contentBuff = cBuff;
|
||||||
} else {
|
} else {
|
||||||
this.handle = new RandomAccessFile(dbFile, "r");
|
this.handle = new RandomAccessFile(xdbFile, "r");
|
||||||
this.vectorIndex = vectorIndex;
|
this.vectorIndex = vectorIndex;
|
||||||
this.contentBuff = null;
|
this.contentBuff = null;
|
||||||
}
|
}
|
||||||
|
|
@ -179,13 +190,15 @@ public class Searcher {
|
||||||
return String.format(
|
return String.format(
|
||||||
"%s->{version:%s, xdb:%s, vIndex:%s, cBuffer:%s}",
|
"%s->{version:%s, xdb:%s, vIndex:%s, cBuffer:%s}",
|
||||||
super.toString(),
|
super.toString(),
|
||||||
version.name, xdbPath,
|
version.name, xdbFile == null ? "null" : xdbFile.getAbsolutePath(),
|
||||||
vectorIndex == null ? "null" : String.valueOf(vectorIndex.length),
|
vectorIndex == null ? "null" : String.valueOf(vectorIndex.length),
|
||||||
contentBuff == null ? "null" : String.valueOf(contentBuff.length())
|
contentBuff == null ? "null" : String.valueOf(contentBuff.length())
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---
|
||||||
// --- static util function
|
// --- static util function
|
||||||
|
// --- read xdb header
|
||||||
|
|
||||||
public static Header loadHeader(RandomAccessFile handle) throws IOException {
|
public static Header loadHeader(RandomAccessFile handle) throws IOException {
|
||||||
handle.seek(0);
|
handle.seek(0);
|
||||||
|
|
@ -194,13 +207,23 @@ public class Searcher {
|
||||||
return new Header(buff);
|
return new Header(buff);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Header loadHeaderFromFile(String dbPath) throws IOException {
|
public static Header loadHeaderFromFile(File xdbFile) throws IOException {
|
||||||
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
|
final RandomAccessFile handle = new RandomAccessFile(xdbFile, "r");
|
||||||
final Header header = loadHeader(handle);
|
final Header header = loadHeader(handle);
|
||||||
handle.close();
|
handle.close();
|
||||||
return header;
|
return header;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static Header loadHeaderFromFile(String xdbPath) throws IOException {
|
||||||
|
return loadHeaderFromFile(new File(xdbPath));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Header loadHeaderFromBuffer(LongByteArray cBuffer) throws IOException {
|
||||||
|
return new Header(cBuffer.slice(0, HeaderInfoLength));
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- read xdb vector index
|
||||||
|
|
||||||
public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException {
|
public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException {
|
||||||
handle.seek(HeaderInfoLength);
|
handle.seek(HeaderInfoLength);
|
||||||
int len = VectorIndexRows * VectorIndexCols * VectorIndexSize;
|
int len = VectorIndexRows * VectorIndexCols * VectorIndexSize;
|
||||||
|
|
@ -213,13 +236,24 @@ public class Searcher {
|
||||||
return buff;
|
return buff;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static byte[] loadVectorIndexFromFile(String dbPath) throws IOException {
|
public static byte[] loadVectorIndexFromFile(File xdbFile) throws IOException {
|
||||||
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
|
final RandomAccessFile handle = new RandomAccessFile(xdbFile, "r");
|
||||||
final byte[] vIndex = loadVectorIndex(handle);
|
final byte[] vIndex = loadVectorIndex(handle);
|
||||||
handle.close();
|
handle.close();
|
||||||
return vIndex;
|
return vIndex;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static byte[] loadVectorIndexFromFile(String xdbPath) throws IOException {
|
||||||
|
return loadVectorIndexFromFile(new File(xdbPath));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] loadVectorIndexFromBuffer(LongByteArray cBuffer) throws IOException {
|
||||||
|
final int len = VectorIndexRows * VectorIndexCols * VectorIndexSize;
|
||||||
|
return cBuffer.slice(HeaderInfoLength, len);
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- read xdb content
|
||||||
|
|
||||||
public static LongByteArray loadContent(RandomAccessFile handle) throws IOException {
|
public static LongByteArray loadContent(RandomAccessFile handle) throws IOException {
|
||||||
handle.seek(0);
|
handle.seek(0);
|
||||||
// check the length and do the buff load
|
// check the length and do the buff load
|
||||||
|
|
@ -239,13 +273,56 @@ public class Searcher {
|
||||||
return byteArray;
|
return byteArray;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static LongByteArray loadContentFromFile(String dbPath) throws IOException {
|
public static LongByteArray loadContentFromFile(File xdbFile) throws IOException {
|
||||||
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
|
final RandomAccessFile handle = new RandomAccessFile(xdbFile, "r");
|
||||||
final LongByteArray content = loadContent(handle);
|
final LongByteArray content = loadContent(handle);
|
||||||
handle.close();
|
handle.close();
|
||||||
return content;
|
return content;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static LongByteArray loadContentFromFile(String xdbPath) throws IOException {
|
||||||
|
return loadContentFromFile(new File(xdbPath));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static LongByteArray loadContentFromInputStream(InputStream is) throws IOException {
|
||||||
|
final LongByteArray byteArray = new LongByteArray();
|
||||||
|
while (true) {
|
||||||
|
boolean done = false;
|
||||||
|
|
||||||
|
// read at most MAX_WRITE_BYTES bytes
|
||||||
|
int rLen, tBytes = 0;
|
||||||
|
final byte[] buff = new byte[MAX_WRITE_BYTES];
|
||||||
|
while (true) {
|
||||||
|
rLen = is.read(buff, tBytes, buff.length - tBytes);
|
||||||
|
if (rLen == -1) {
|
||||||
|
// reach the end of the stream
|
||||||
|
done = true;
|
||||||
|
break;
|
||||||
|
} else if (rLen == 0) {
|
||||||
|
// the entire buff was filled
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
tBytes += rLen;
|
||||||
|
}
|
||||||
|
|
||||||
|
// check and copy the buffer with its actual filled bytes
|
||||||
|
if (tBytes == buff.length) {
|
||||||
|
byteArray.append(buff);
|
||||||
|
} else {
|
||||||
|
final byte[] nBuff = new byte[tBytes];
|
||||||
|
System.arraycopy(buff, 0, nBuff, 0, tBytes);
|
||||||
|
byteArray.append(nBuff);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (done) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return byteArray;
|
||||||
|
}
|
||||||
|
|
||||||
// --- verify util function
|
// --- verify util function
|
||||||
|
|
||||||
// Verify if the current Searcher could be used to search the specified xdb file.
|
// Verify if the current Searcher could be used to search the specified xdb file.
|
||||||
|
|
@ -254,9 +331,7 @@ public class Searcher {
|
||||||
//
|
//
|
||||||
// @Note: You Just need to check this ONCE when the service starts
|
// @Note: You Just need to check this ONCE when the service starts
|
||||||
// Or use another process (eg, A command) to check once Just to confirm the suitability.
|
// Or use another process (eg, A command) to check once Just to confirm the suitability.
|
||||||
public static void verify(RandomAccessFile handle) throws IOException, XdbException {
|
public static void verify(Header header, long fileBytes) throws IOException, XdbException {
|
||||||
final Header header = loadHeader(handle);
|
|
||||||
|
|
||||||
// get the runtime ptr bytes
|
// get the runtime ptr bytes
|
||||||
int runtimePtrBytes = 0;
|
int runtimePtrBytes = 0;
|
||||||
if (header.version == STRUCTURE_20) {
|
if (header.version == STRUCTURE_20) {
|
||||||
|
|
@ -270,15 +345,23 @@ public class Searcher {
|
||||||
// 1, confirm the xdb file size
|
// 1, confirm the xdb file size
|
||||||
// to ensure that the maximum file pointer does not overflow
|
// to ensure that the maximum file pointer does not overflow
|
||||||
final long maxFilePtr = (1L << (runtimePtrBytes * 8)) - 1;
|
final long maxFilePtr = (1L << (runtimePtrBytes * 8)) - 1;
|
||||||
if (handle.length() > maxFilePtr) {
|
if (fileBytes > maxFilePtr) {
|
||||||
throw new XdbException("xdb file exceeds the maximum supported bytes: "+maxFilePtr+"");
|
throw new XdbException("xdb file exceeds the maximum supported bytes: "+maxFilePtr+"");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void verifyFromFile(String dbFile) throws IOException, XdbException {
|
public static void verify(RandomAccessFile handle) throws IOException, XdbException {
|
||||||
final RandomAccessFile handle = new RandomAccessFile(dbFile, "r");
|
verify(loadHeader(handle), handle.length());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void verifyFromFile(File xdbFile) throws IOException, XdbException {
|
||||||
|
final RandomAccessFile handle = new RandomAccessFile(xdbFile, "r");
|
||||||
verify(handle);
|
verify(handle);
|
||||||
handle.close();
|
handle.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static void verifyFromFile(String xdbPath) throws IOException, XdbException {
|
||||||
|
verifyFromFile(new File(xdbPath));
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
package org.lionsoul.ip2region.service;
|
package org.lionsoul.ip2region.service;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.security.CodeSource;
|
import java.security.CodeSource;
|
||||||
|
|
||||||
|
|
@ -21,9 +23,9 @@ public class ConfigTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testBuildV4Config() throws IOException, XdbException {
|
public void testBuildV4Config() throws IOException, XdbException, InvalidConfigException {
|
||||||
final Config v4Config = Config.custom()
|
final Config v4Config = Config.custom()
|
||||||
.setCachePolicy(Config.BufferCache)
|
.setCachePolicy(Config.VIndexCache)
|
||||||
.setXdbPath(getDataPath("ip2region_v4.xdb"))
|
.setXdbPath(getDataPath("ip2region_v4.xdb"))
|
||||||
.setSearchers(20)
|
.setSearchers(20)
|
||||||
.asV4();
|
.asV4();
|
||||||
|
|
@ -31,7 +33,29 @@ public class ConfigTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testBuildV6Config() throws IOException, XdbException {
|
public void testBuildV4ConfigFromFile() throws IOException, XdbException, InvalidConfigException {
|
||||||
|
final Config v4Config = Config.custom()
|
||||||
|
.setCachePolicy(Config.BufferCache)
|
||||||
|
.setXdbFile(new File(getDataPath("ip2region_v4.xdb")))
|
||||||
|
.setSearchers(20)
|
||||||
|
.asV4();
|
||||||
|
log.debugf("builded config: %s", v4Config);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testBuildV4ConfigFromInputStream() throws IOException, XdbException, InvalidConfigException {
|
||||||
|
final Config v4Config = Config.custom()
|
||||||
|
.setCachePolicy(Config.BufferCache)
|
||||||
|
.setXdbInputStream(new FileInputStream(getDataPath("ip2region_v4.xdb")))
|
||||||
|
.setSearchers(20)
|
||||||
|
.asV4();
|
||||||
|
log.debugf("builded config: %s", v4Config);
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- IPv6
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testBuildV6Config() throws IOException, XdbException, InvalidConfigException {
|
||||||
final Config v4Config = Config.custom()
|
final Config v4Config = Config.custom()
|
||||||
.setCachePolicy(Config.VIndexCache)
|
.setCachePolicy(Config.VIndexCache)
|
||||||
.setXdbPath(getDataPath("ip2region_v6.xdb"))
|
.setXdbPath(getDataPath("ip2region_v6.xdb"))
|
||||||
|
|
@ -39,4 +63,25 @@ public class ConfigTest {
|
||||||
.asV6();
|
.asV6();
|
||||||
log.debugf("builded config: %s", v4Config);
|
log.debugf("builded config: %s", v4Config);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testBuildV6ConfigFromFile() throws IOException, XdbException, InvalidConfigException {
|
||||||
|
final Config v4Config = Config.custom()
|
||||||
|
.setCachePolicy(Config.BufferCache)
|
||||||
|
.setXdbFile(new File(getDataPath("ip2region_v6.xdb")))
|
||||||
|
.setSearchers(20)
|
||||||
|
.asV6();
|
||||||
|
log.debugf("builded config: %s", v4Config);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testBuildV6ConfigFromInputStream() throws IOException, XdbException, InvalidConfigException {
|
||||||
|
final Config v4Config = Config.custom()
|
||||||
|
.setCachePolicy(Config.BufferCache)
|
||||||
|
.setXdbInputStream(new FileInputStream(getDataPath("ip2region_v6.xdb")))
|
||||||
|
.setSearchers(20)
|
||||||
|
.asV6();
|
||||||
|
log.debugf("builded config: %s", v4Config);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,7 @@ public class Ip2RegionTest {
|
||||||
private static final Log log = Log.getLogger(Ip2RegionTest.class).setLevel(Log.DEBUG);
|
private static final Log log = Log.getLogger(Ip2RegionTest.class).setLevel(Log.DEBUG);
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void TestConfigCreate() throws IOException, XdbException, InetAddressException, InterruptedException {
|
public void TestConfigCreate() throws IOException, XdbException, InetAddressException, InterruptedException, InvalidConfigException {
|
||||||
final Config v4Config = Config.custom()
|
final Config v4Config = Config.custom()
|
||||||
.setCachePolicy(Config.NoCache)
|
.setCachePolicy(Config.NoCache)
|
||||||
.setSearchers(10)
|
.setSearchers(10)
|
||||||
|
|
@ -46,7 +46,7 @@ public class Ip2RegionTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void TestPathCreate() throws InetAddressException, IOException, XdbException, InterruptedException {
|
public void TestPathCreate() throws InetAddressException, IOException, XdbException, InterruptedException, InvalidConfigException {
|
||||||
byte[] v4Bytes = Util.parseIP("113.92.157.29");
|
byte[] v4Bytes = Util.parseIP("113.92.157.29");
|
||||||
byte[] v6Bytes = Util.parseIP("240e:3b7:3272:d8d0:db09:c067:8d59:539e");
|
byte[] v6Bytes = Util.parseIP("240e:3b7:3272:d8d0:db09:c067:8d59:539e");
|
||||||
final Ip2Region ip2Region = Ip2Region.create(ConfigTest.getDataPath("ip2region_v4.xdb"), ConfigTest.getDataPath("ip2region_v6.xdb"));
|
final Ip2Region ip2Region = Ip2Region.create(ConfigTest.getDataPath("ip2region_v4.xdb"), ConfigTest.getDataPath("ip2region_v6.xdb"));
|
||||||
|
|
@ -63,7 +63,7 @@ public class Ip2RegionTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void TestInMemSearch() throws IOException, XdbException, InetAddressException, InterruptedException {
|
public void TestInMemSearch() throws IOException, XdbException, InetAddressException, InterruptedException, InvalidConfigException {
|
||||||
final Config v4Config = Config.custom()
|
final Config v4Config = Config.custom()
|
||||||
.setCachePolicy(Config.BufferCache)
|
.setCachePolicy(Config.BufferCache)
|
||||||
.setXdbPath(ConfigTest.getDataPath("ip2region_v4.xdb"))
|
.setXdbPath(ConfigTest.getDataPath("ip2region_v4.xdb"))
|
||||||
|
|
@ -90,7 +90,7 @@ public class Ip2RegionTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void TestConcurrentCall() throws IOException, XdbException, InetAddressException, InterruptedException {
|
public void TestConcurrentCall() throws IOException, XdbException, InetAddressException, InterruptedException, InvalidConfigException {
|
||||||
final Config v4Config = Config.custom()
|
final Config v4Config = Config.custom()
|
||||||
.setCachePolicy(Config.VIndexCache)
|
.setCachePolicy(Config.VIndexCache)
|
||||||
.setSearchers(15)
|
.setSearchers(15)
|
||||||
|
|
@ -144,7 +144,7 @@ public class Ip2RegionTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void TestV4Only() throws IOException, XdbException, InetAddressException, InterruptedException {
|
public void TestV4Only() throws IOException, XdbException, InetAddressException, InterruptedException, InvalidConfigException {
|
||||||
final Config v4Config = Config.custom()
|
final Config v4Config = Config.custom()
|
||||||
.setCachePolicy(Config.NoCache)
|
.setCachePolicy(Config.NoCache)
|
||||||
.setXdbPath(ConfigTest.getDataPath("ip2region_v4.xdb"))
|
.setXdbPath(ConfigTest.getDataPath("ip2region_v4.xdb"))
|
||||||
|
|
|
||||||
|
|
@ -32,6 +32,30 @@ public class SearcherPoolTest {
|
||||||
log.debugf("v4 searcher pool closed gracefully");
|
log.debugf("v4 searcher pool closed gracefully");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testInMemV4SearcherPool() throws Exception {
|
||||||
|
final Config v4Config = Config.custom()
|
||||||
|
.setCachePolicy(Config.FullCache)
|
||||||
|
.setSearchers(5)
|
||||||
|
.setXdbPath(ConfigTest.getDataPath("ip2region_v4.xdb"))
|
||||||
|
.asV4();
|
||||||
|
|
||||||
|
|
||||||
|
final String ipStr = "58.250.36.41";
|
||||||
|
final SearcherPool v4Pool = SearcherPool.create(v4Config);
|
||||||
|
for (int i = 0; i < 20; i++) {
|
||||||
|
final Searcher searcher = v4Pool.borrowSearcher();
|
||||||
|
log.debugf("borrowed searcher %d: %s", i, searcher.toString());
|
||||||
|
final String region = searcher.search(ipStr);
|
||||||
|
log.debugf("search(%s)=%s", ipStr, region);
|
||||||
|
v4Pool.returnSearcher(searcher);
|
||||||
|
log.debugf("return searcher %d", i);
|
||||||
|
}
|
||||||
|
|
||||||
|
v4Pool.close();
|
||||||
|
log.debugf("v4 searcher pool closed gracefully");
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testV6SearcherPool() throws Exception {
|
public void testV6SearcherPool() throws Exception {
|
||||||
final Config v6Config = Config.custom()
|
final Config v6Config = Config.custom()
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue