Merge pull request #1 from gongzhengyang/feature-rust-binding
Feature rust binding
This commit is contained in:
gongzhengyang
GitHub
commit
777c98434d
|
|
@ -1,2 +1,2 @@
|
||||||
[workspace]
|
[workspace]
|
||||||
members = ["bin", "search"]
|
members = ["example", "ip2region2"]
|
||||||
|
|
|
||||||
|
|
@ -1,137 +1,359 @@
|
||||||
# `ip2region xdb rust` 查询客户端实现
|
# `ip2region xdb rust` 查询客户端实现
|
||||||
|
|
||||||
|
# 实现效果
|
||||||
|
|
||||||
|
得益于`xdb`数据存储格式设计以及`rust`编译器的高度代码优化
|
||||||
|
|
||||||
|
- 实现单核`CPU`下接近每秒千万级别的查询，如果是4核8线这样的`CPU`，采用`tokio`异步运行时，可以达到接近每秒4千万查询速度，查询速度取决于`CPU`物理核睿频频率
|
||||||
|
- 达到查询稳定在`100-150ns/op`
|
||||||
|
- `ip2region.xdb`文件会直接加载到内存，整个程序运行时候占用内存`13M`左右，即使是多线程或者异步运行时下面的高并发查询也是稳定在这个内存大小
|
||||||
|
- 只会加载一次数据，多线程安全，可以自由使用`tokio`异步运行时或者标准库的多线程`std::thread`
|
||||||
|
|
||||||
|
# 缓存方式说明
|
||||||
|
|
||||||
|
由于基于文件的查询以及缓存`VectorIndex`索引在并发较高(比如每秒上百并发)的情况下，每次查询都会从磁盘加载`ip2region.xdb`文件进入内存，由此会产生很高的磁盘`IO`以及极大的内存占用，所以决定做一次减法，不对这两种缓存进行开发，只提供缓存整个`xdb`文件的方式，以此实现最小的并发查询内存开销以及极限`CPU`性能压榨
|
||||||
|
|
||||||
# 使用方式
|
# 使用方式
|
||||||
|
|
||||||
### 缓存整个 `xdb` 数据
|
使用`cargo`新建一个项目，比如`cargo new ip-test`
|
||||||
|
|
||||||
预先加载整个` ip2region.xdb` 到内存，完全基于内存查询，该方式线程安全，采用`once_cell::sync::OnceCell`，只会加载一次数据，多线程安全，可以自由使用`tokio`异步运行时或者标准库的多线程`std::thread`
|
同时把`ip2region.xdb`文件也移动到该项目根路径下，或者不移动，下面示例编译的时候注意调整`xdb_filepath`的参数值
|
||||||
|
|
||||||
配置`Cargo.toml`如下
|
配置`Cargo.toml`的`[dependencies]`如下
|
||||||
|
|
||||||
```toml
|
```toml
|
||||||
[dependencies]
|
[dependencies]
|
||||||
search = { git = "https://github.com/lionsoul2014/ip2region.git", branch = "master" }
|
search = { git = "https://github.com/lionsoul2014/ip2region.git", branch = "master" }
|
||||||
# 如果要在异步环境下使用，需要加上如下依赖
|
# 用于生成随机数
|
||||||
|
rand = "0.8"
|
||||||
|
# 用于初始化日志打印
|
||||||
|
tracing = "0.1"
|
||||||
|
tracing-subscriber = "0.3.14"
|
||||||
|
# 异步运行时
|
||||||
tokio = { version = "1", features = ["full"]}
|
tokio = { version = "1", features = ["full"]}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### 基本使用示例
|
||||||
|
|
||||||
编写`main.rs`
|
编写`main.rs`
|
||||||
|
|
||||||
**需要使用`XDB_FILEPATH`指定`ip2region.xdb`文件的路径**，该参数可以使用相对路径或者绝对路径，如果使用相对路径报错，请修改为绝对路径
|
|
||||||
|
|
||||||
```rust
|
```rust
|
||||||
use std::env;
|
use std::net::Ipv4Addr;
|
||||||
|
use std::thread;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
#[tokio::main]
|
use ip2region2::{search_by_ip, searcher_init};
|
||||||
async fn main() {
|
|
||||||
env::set_var(
|
fn main() {
|
||||||
"XDB_FILEPATH",
|
// 配置输出日志信息
|
||||||
"../data/ip2region.xdb",
|
tracing_subscriber::fmt::init();
|
||||||
);
|
|
||||||
// search_by_ip的参数可以是u32类型，字符串IP类型，字符串数字类型
|
// 初始化加载xdb文件
|
||||||
|
let xdb_filepath = "./ip2region.xdb";
|
||||||
|
searcher_init(Some(xdb_filepath.to_owned()));
|
||||||
|
// 如果../data或者../../data或者../../../data下面有对应的ip2region.xdb文件
|
||||||
|
// 初始化函数可以直接调用如下
|
||||||
|
// searcher_init(None);
|
||||||
|
|
||||||
|
println!("\n测试多类型查询");
|
||||||
|
println!("{}", search_by_ip("9999999").unwrap());
|
||||||
|
println!("{}", search_by_ip("1.0.1.0").unwrap());
|
||||||
|
println!("{}", search_by_ip(9999999).unwrap());
|
||||||
|
println!("{:?}", search_by_ip(Ipv4Addr::from(3333333)));
|
||||||
|
|
||||||
|
println!("\n测试多线程初始化以及多线程查询");
|
||||||
for i in 1..5 {
|
for i in 1..5 {
|
||||||
tokio::spawn(async move {
|
thread::spawn(move || {
|
||||||
// u32
|
// 再次初始化是没什么效果的
|
||||||
println!("{:?}", search::search_by_ip(i));
|
searcher_init(Some(xdb_filepath.to_owned()));
|
||||||
|
println!("in thread {i} {:?}", search_by_ip(rand::random::<u32>()));
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
// ip str
|
// 等待多线程执行结束
|
||||||
println!("{:?}", search::search_by_ip("1.0.1.0"));
|
thread::sleep(Duration::from_secs(1));
|
||||||
|
|
||||||
// u32 str
|
let count = 10_000_000;
|
||||||
let ip_u32 = 1 << 24 | 1 << 8;
|
print!("\n计算千万数据总共耗时: ");
|
||||||
println!("{:?}", search::search_by_ip(ip_u32.to_string().as_str()));
|
let now = Instant::now();
|
||||||
|
for _ in 0..count {
|
||||||
|
search_by_ip(rand::random::<u32>()).unwrap();
|
||||||
|
}
|
||||||
|
println!("{:?}, ave: {:?}", now.elapsed(), now.elapsed()/count);
|
||||||
|
|
||||||
|
print!("\n计算千万数据，每次迭代都统计耗时的总共耗时: ");
|
||||||
|
let mut total = Duration::from_secs(0);
|
||||||
|
for _ in 0..count {
|
||||||
|
let current = Instant::now();
|
||||||
|
search_by_ip(rand::random::<u32>()).unwrap();
|
||||||
|
total += current.elapsed();
|
||||||
|
}
|
||||||
|
println!("{:?}, ave: {:?}", total, total/count);
|
||||||
|
|
||||||
|
print!("\n计算空迭代耗时: ");
|
||||||
|
let now = Instant::now();
|
||||||
|
for _ in 0..count {}
|
||||||
|
println!("{:?}", now.elapsed());
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
进行测试
|
进行测试如下，需要指定`RUST_LOG`参数打印日志
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
$ cargo run
|
➜ RUST_LOG=debug cargo run -r
|
||||||
init xdb searcher at ../data/ip2region.xdb
|
Compiling ip-test v0.1.0 (/home/gong/rust-work/ip-test)
|
||||||
Ok("中国|0|福建省|福州市|电信")
|
Finished release [optimized] target(s) in 0.27s
|
||||||
Ok("中国|0|福建省|福州市|电信")
|
Running `target/release/ip-test`
|
||||||
Ok("0|0|0|内网IP|内网IP")
|
2022-12-24T03:31:22.921958Z DEBUG ip2region2::searcher: load xdb searcher file at ./ip2region.xdb
|
||||||
Ok("0|0|0|内网IP|内网IP")
|
|
||||||
Ok("0|0|0|内网IP|内网IP")
|
测试多类型查询
|
||||||
|
0|0|0|内网IP|内网IP
|
||||||
|
中国|0|福建省|福州市|电信
|
||||||
|
0|0|0|内网IP|内网IP
|
||||||
Ok("0|0|0|内网IP|内网IP")
|
Ok("0|0|0|内网IP|内网IP")
|
||||||
|
|
||||||
|
测试多线程初始化以及多线程查询
|
||||||
|
in thread 1 Ok("美国|0|新墨西哥|0|康卡斯特")
|
||||||
|
in thread 3 Ok("0|0|0|内网IP|内网IP")
|
||||||
|
in thread 2 Ok("土耳其|0|Ankara|0|0")
|
||||||
|
in thread 4 Ok("爱尔兰|0|Cork|0|0")
|
||||||
|
|
||||||
|
计算千万数据总共耗时: 1.176241972s, ave: 117ns
|
||||||
|
|
||||||
|
计算千万数据，每次迭代都统计耗时的总共耗时: 1.407956755s, ave: 140ns
|
||||||
|
|
||||||
|
计算空迭代耗时: 40ns
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### `tokio`使用示例
|
||||||
|
|
||||||
|
```rust
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use tokio::sync::mpsc;
|
||||||
|
|
||||||
|
use ip2region2::{search_by_ip, searcher_init};
|
||||||
|
|
||||||
|
#[tokio::main]
|
||||||
|
async fn main() {
|
||||||
|
// 配置输出debug 信息
|
||||||
|
tracing_subscriber::fmt::init();
|
||||||
|
searcher_init(Some("./ip2region.xdb".to_owned()));
|
||||||
|
let main_now = Instant::now();
|
||||||
|
let (tx, mut rx) = mpsc::channel(10);
|
||||||
|
for i in 0..6 {
|
||||||
|
let tx = tx.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let now = Instant::now();
|
||||||
|
let count = 10_000_000;
|
||||||
|
for _ in 0..count {
|
||||||
|
search_by_ip(rand::random::<u32>()).unwrap();
|
||||||
|
}
|
||||||
|
let message = format!(
|
||||||
|
"tokio spawn {i} over cost: {:?}, ave: {:?}",
|
||||||
|
now.elapsed(),
|
||||||
|
now.elapsed() / count
|
||||||
|
);
|
||||||
|
tx.send(message).await.unwrap();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
drop(tx);
|
||||||
|
while let Some(message) = rx.recv().await {
|
||||||
|
println!("{}", message);
|
||||||
|
}
|
||||||
|
println!("总共耗时: {:?}", main_now.elapsed());
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
开始执行测试
|
||||||
|
|
||||||
|
```shell
|
||||||
|
$ RUST_LOG=debug cargo run -r
|
||||||
|
Compiling ip-test v0.1.0 (/home/gong/rust-work/ip-test)
|
||||||
|
Finished release [optimized] target(s) in 0.51s
|
||||||
|
Running `target/release/ip-test`
|
||||||
|
2022-12-24T04:05:32.876664Z DEBUG ip2region2::searcher: load xdb searcher file at ./ip2region.xdb
|
||||||
|
tokio spawn 4 over cost: 1.133448675s, ave: 113ns
|
||||||
|
tokio spawn 2 over cost: 1.133938619s, ave: 113ns
|
||||||
|
tokio spawn 1 over cost: 1.136872027s, ave: 113ns
|
||||||
|
tokio spawn 5 over cost: 1.173464286s, ave: 117ns
|
||||||
|
tokio spawn 0 over cost: 1.197527014s, ave: 119ns
|
||||||
|
tokio spawn 3 over cost: 1.26446099s, ave: 126ns
|
||||||
|
总共耗时: 1.264631935s
|
||||||
|
```
|
||||||
|
|
||||||
|
# `binding/rust`路径下面的结构说明
|
||||||
|
|
||||||
|
`ip2region2`
|
||||||
|
|
||||||
|
- 封装了`ip`到`region`的函数
|
||||||
|
- 里面包含单元测试和`benchmark`测试
|
||||||
|
|
||||||
|
`example`
|
||||||
|
|
||||||
|
- 包含了命令行可执行文件生成的源码程序
|
||||||
|
- 作为一个用于`rust`的开发集成例子
|
||||||
|
|
||||||
|
开始编译之后会生成如下
|
||||||
|
|
||||||
|
`target`
|
||||||
|
|
||||||
|
- 文件夹存放编译之后的文件以及编译产生的临时文件与缓存
|
||||||
|
|
||||||
|
`Cargo.lock`
|
||||||
|
|
||||||
|
- 固定`rust`第三方库的版本
|
||||||
|
|
||||||
|
编译生成的文件全部在`.gitignore`中有标识，不会被提交
|
||||||
|
|
||||||
# 编译程序
|
# 编译程序
|
||||||
|
|
||||||
通过如下方式编译得到 `ip2region` 可执行程序
|
切换到 `ip2region/binding/rust` 路径，执行如下命令
|
||||||
|
|
||||||
切换到 `rust binding` 根目录，执行如下命令
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
➜ cargo build -r
|
$ cargo build -r
|
||||||
```
|
```
|
||||||
|
|
||||||
生成的二进制文件会在`./target/release/ip2region`位置
|
生成的二进制文件会在`./target/release/rust-example`位置
|
||||||
|
|
||||||
# 查询测试
|
# 查询测试
|
||||||
|
|
||||||
通过 `./target/release/ip2region` 命令来测试查询
|
切换到 `ip2region/binding/rust` 路径，执行如下命令
|
||||||
```
|
|
||||||
➜ ./target/release/ip2region --help
|
`help`输出如下
|
||||||
you can set environment XDB_FILEPATH=../data/ip2region or just use --xdb in command
|
|
||||||
Usage: ip2region [OPTIONS]
|
```shell
|
||||||
|
$ ./target/release/rust-example query --help
|
||||||
|
query test
|
||||||
|
|
||||||
|
Usage: rust-example query [OPTIONS]
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
--xdb <xdb>
|
--db <db> the xdb filepath, you can set this field like ../data/ip2region.xdb,if you dont set,if will detect xdb file on ../data/ip2region.xdb, ../../data/ip2region.xdb, ../../../data/ip2region.xdb if exists
|
||||||
the xdb filepath, you can set this field like ../data/ip2region.xdb
|
-h, --help Print help information
|
||||||
-h, --help
|
|
||||||
Print help information (use `-h` for a summary)
|
|
||||||
-V, --version
|
|
||||||
Print version information
|
|
||||||
```
|
```
|
||||||
|
|
||||||
命令行指定参数进行查询测试，输入 `ip` 地址或者一个`u32`类型的数字进行查询即可，输入 `quit` 退出测试程序
|
执行测试，使用默认`data/ip2region.txt`
|
||||||
```bash
|
|
||||||
➜ ./target/release/ip2region --xdb=../../data/ip2region.xdb
|
```shell
|
||||||
init xdb searcher at ../../data/ip2region.xdb
|
$ ./target/release/rust-example query --db=../../data/ip2region.xdb
|
||||||
ip2region xdb searcher test program, type `quit` to exit
|
ip2region xdb searcher test program, type `quit` or `Ctrl + c` to exit
|
||||||
|
ip2region>> 123123123
|
||||||
|
region: Ok("美国|0|0|0|0"), took: 4.94µs
|
||||||
ip2region>> 1.1.1.1
|
ip2region>> 1.1.1.1
|
||||||
region: Ok("澳大利亚|0|0|0|0"), took: 4.227µs
|
region: Ok("澳大利亚|0|0|0|0"), took: 2.057µs
|
||||||
ip2region>> 2.2.2.2
|
ip2region>> 2.2.2.2
|
||||||
region: Ok("法国|0|0|0|橘子电信"), took: 4.495µs
|
region: Ok("法国|0|0|0|橘子电信"), took: 4.294µs
|
||||||
ip2region>> 222222222
|
ip2region>>
|
||||||
region: Ok("美国|0|康涅狄格|0|0"), took: 4.048µs
|
|
||||||
```
|
```
|
||||||
|
|
||||||
或者使用环境变量
|
这边发现每次查询的消耗时间都超过`1µs`，和开头所说的纳秒级查询不一致啊，这个是由于`rust`的标准库封装的`use std::time::Instant`对象是调用系统底层函数实现的，导致会有微秒级别的误差
|
||||||
|
|
||||||
```shell
|
可以试着找一个新的项目
|
||||||
➜ XDB_FILEPATH=../../data/ip2region.xdb ./target/release/ip2region
|
|
||||||
init xdb searcher at ../../data/ip2region.xdb
|
在`main.rs`写入如下
|
||||||
ip2region xdb searcher test program, type `quit` to exit
|
|
||||||
ip2region>> 2.2.2.2
|
```rust
|
||||||
region: Ok("法国|0|0|0|橘子电信"), took: 4.458µs
|
use std::thread;
|
||||||
ip2region>> 4.4.4.5
|
use std::time::{Instant, Duration};
|
||||||
region: Ok("美国|0|0|0|Level3"), took: 4.847µs
|
|
||||||
|
fn main() {
|
||||||
|
let now = Instant::now();
|
||||||
|
thread::sleep(Duration::from_secs(3));
|
||||||
|
println!("{:?}", now.elapsed());
|
||||||
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
# 单元测试
|
执行命令如下，发现毫秒级别是没什么问题都，微秒和纳秒上面是存在误差的，详情可以参考[`rust`标准库文档的`time::Instant`章](https://rustwiki.org/zh-CN/std/time/struct.Instant.html)
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
➜ XDB_FILEPATH=../../../data/ip2region.xdb cargo test
|
$ cargo run -r
|
||||||
|
Finished release [optimized] target(s) in 0.03s
|
||||||
|
Running `target/release/ip-test`
|
||||||
|
3.000197389s
|
||||||
```
|
```
|
||||||
|
|
||||||
# `bench` 测试
|
# `bench`测试
|
||||||
|
|
||||||
通过 `cargo bench` 命令来进行自动 `bench` 测试，一方面确保程序和 `xdb` 文件都没有错误，另一方面通过大量的查询得到平均查询性能
|
测试平均性能
|
||||||
|
|
||||||
在不同机器上面的测试性能时间是不一样的，如下是在机器`CPU`是`Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz`，内存`DDR4 32G`下面的测试结果
|
切换到 `ip2region/binding/rust` 路径，执行如下命令
|
||||||
|
|
||||||
|
`help`输出如下
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
➜ XDB_FILEPATH=../../../data/ip2region.xdb cargo bench
|
$ ./target/release/rust-example bench --help
|
||||||
|
bench test
|
||||||
|
|
||||||
|
Usage: rust-example bench [OPTIONS] --src <src>
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--src <src> set this to specific source bench file
|
||||||
|
--db <db> the xdb filepath, you can set this field like ../data/ip2region.xdb,if you dont set,if will detect xdb file on ../data/ip2region.xdb, ../../data/ip2region.xdb, ../../../data/ip2region.xdb if exists
|
||||||
|
-h, --help Print help information
|
||||||
|
```
|
||||||
|
|
||||||
|
使用默认的`ip2region`和`ip.merge.txt`
|
||||||
|
|
||||||
|
```shell
|
||||||
|
$ ./target/release/rust-example bench --src=../../data/ip.merge.txt --db=../../data/ip2region.xdb
|
||||||
|
Bench finished, total: 3419220,took: 519.820535ms ,cost: 152ns/op
|
||||||
|
```
|
||||||
|
|
||||||
|
# `binding/rust`后续维护须知
|
||||||
|
|
||||||
|
`bingd/rust`编写了单元测试，后续开发需要保证单元测试正常
|
||||||
|
|
||||||
|
切换到 `ip2region/binding/rust` 路径，执行如下命令
|
||||||
|
|
||||||
|
```shell
|
||||||
|
$ cargo test
|
||||||
|
```
|
||||||
|
|
||||||
|
需要保证查询速度不会有大幅降低，希望有朝一日，远方的朋友可以再优化一下，实现几十纳秒级别的查询速度
|
||||||
|
|
||||||
|
下面是`ip2region2`库的第一版`benchmark`结果
|
||||||
|
|
||||||
|
重点关注如下
|
||||||
|
|
||||||
|
`search_by_ip_bench `
|
||||||
|
|
||||||
|
- 查询`ip`的实际调用函数
|
||||||
|
|
||||||
|
`get_block_by_size_bench`
|
||||||
|
|
||||||
|
- 获取并且计算偏移值，和其他`binding`下的实现的`getLong`和`getShort`相似
|
||||||
|
- 该函数会被`search_by_ip`多次调用，所以被标注为`#[inline]`使用内联优化，以此来消除函数调用产生的压栈开销
|
||||||
|
|
||||||
|
```shell
|
||||||
|
$ cargo bench
|
||||||
|
// --snip---
|
||||||
|
search_by_ip_bench time: [116.99 ns 119.52 ns 122.31 ns]
|
||||||
|
change: [-8.1930% -5.8295% -3.3029%] (p = 0.00 < 0.05)
|
||||||
|
Performance has improved.
|
||||||
|
Found 4 outliers among 100 measurements (4.00%)
|
||||||
|
4 (4.00%) high mild
|
||||||
|
|
||||||
|
get_block_by_size_bench time: [5.2388 ns 5.2784 ns 5.3229 ns]
|
||||||
|
change: [-6.2649% -4.3559% -2.5539%] (p = 0.00 < 0.05)
|
||||||
|
Performance has improved.
|
||||||
|
Found 7 outliers among 100 measurements (7.00%)
|
||||||
|
3 (3.00%) high mild
|
||||||
|
4 (4.00%) high severe
|
||||||
|
|
||||||
|
get_full_cache_bench time: [1.4800 ns 1.5034 ns 1.5325 ns]
|
||||||
|
change: [-17.984% -13.664% -9.1681%] (p = 0.00 < 0.05)
|
||||||
|
Performance has improved.
|
||||||
|
Found 15 outliers among 100 measurements (15.00%)
|
||||||
|
3 (3.00%) high mild
|
||||||
|
12 (12.00%) high severe
|
||||||
|
|
||||||
|
get_vec_index_cache_bench
|
||||||
|
time: [1.7578 ns 1.7757 ns 1.7961 ns]
|
||||||
|
change: [-7.5169% -4.5088% -1.7147%] (p = 0.00 < 0.05)
|
||||||
|
Performance has improved.
|
||||||
|
Found 5 outliers among 100 measurements (5.00%)
|
||||||
|
3 (3.00%) high mild
|
||||||
|
2 (2.00%) high severe
|
||||||
// --snip--
|
// --snip--
|
||||||
Running benches/search.rs (target/release/deps/search-9614305a566885c4)
|
|
||||||
Benchmarking ip_search_bench: Warming up for 3.0000 sinit xdb searcher at ../../../data/ip2region.xdb
|
|
||||||
ip_search_bench time: [120.84 ns 122.91 ns 125.20 ns]
|
|
||||||
change: [-3.3346% -1.2786% +0.8027%] (p = 0.23 > 0.05)
|
|
||||||
No change in performance detected.
|
|
||||||
Found 6 outliers among 100 measurements (6.00%)
|
|
||||||
6 (6.00%) high mild
|
|
||||||
```
|
```
|
||||||
|
|
||||||
可以看到上面的`ip_search_bench time`一行的参数表示是左右值分别显示置信区间的下限和上限，中间值显示 `Criterion.rs` 对基准程序每次迭代所用时间的最佳估计
|
|
||||||
|
|
|
||||||
|
|
@ -1,14 +0,0 @@
|
||||||
use clap::{arg, ArgMatches, Command};
|
|
||||||
|
|
||||||
pub fn get_matches() -> ArgMatches {
|
|
||||||
Command::new("ip2region")
|
|
||||||
.version("0.1")
|
|
||||||
.about("ip2region bin program")
|
|
||||||
.long_about(
|
|
||||||
"you can set environment XDB_FILEPATH=../data/ip2region or just use --xdb in command",
|
|
||||||
)
|
|
||||||
.arg(
|
|
||||||
arg!(--xdb <xdb> "the xdb filepath, you can set this field like ../data/ip2region.xdb"),
|
|
||||||
)
|
|
||||||
.get_matches()
|
|
||||||
}
|
|
||||||
|
|
@ -1,31 +0,0 @@
|
||||||
use std::env;
|
|
||||||
use std::io::Write;
|
|
||||||
use std::time::Instant;
|
|
||||||
|
|
||||||
mod cmd;
|
|
||||||
|
|
||||||
fn main() {
|
|
||||||
env::var("XDB_FILEPATH").unwrap_or_else(|_| {
|
|
||||||
let matches = cmd::get_matches();
|
|
||||||
let xdb_filepath = matches
|
|
||||||
.get_one::<String>("xdb")
|
|
||||||
.expect("you must use --xdb in command or set XDB_FILEPATH environment");
|
|
||||||
env::set_var("XDB_FILEPATH", xdb_filepath);
|
|
||||||
xdb_filepath.to_owned()
|
|
||||||
});
|
|
||||||
|
|
||||||
search::global_searcher();
|
|
||||||
println!("ip2region xdb searcher test program, type `quit` or `Ctrl + c` to exit");
|
|
||||||
loop {
|
|
||||||
print!("ip2region>> ");
|
|
||||||
std::io::stdout().flush().unwrap();
|
|
||||||
let mut line = String::new();
|
|
||||||
std::io::stdin().read_line(&mut line).unwrap();
|
|
||||||
if line.contains("quit") {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
let now = Instant::now();
|
|
||||||
let result = search::search_by_ip(line.trim());
|
|
||||||
println!("region: {:?}, took: {:?}", result, now.elapsed());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "ip2region"
|
name = "rust-example"
|
||||||
default-run = "ip2region"
|
default-run = "rust-example"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.66.0"
|
rust-version = "1.66.0"
|
||||||
|
|
@ -10,5 +10,7 @@ license = "Apache-2.0"
|
||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
search = { path = "../search" }
|
ip2region2 = { path = "../ip2region2" }
|
||||||
clap = { version = "4.0" }
|
clap = { version = "4.0" }
|
||||||
|
tracing = "0.1"
|
||||||
|
tracing-subscriber = "0.3.14"
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
use clap::{Arg, ArgMatches, Command};
|
||||||
|
|
||||||
|
pub fn get_matches() -> ArgMatches {
|
||||||
|
let db_arg = Arg::new("db")
|
||||||
|
.long("db")
|
||||||
|
.help("the xdb filepath, you can set this field like \
|
||||||
|
../data/ip2region.xdb,if you dont set,\
|
||||||
|
if will detect xdb file on ../data/ip2region.xdb, ../../data/ip2region.xdb, ../../../data/ip2region.xdb if exists");
|
||||||
|
|
||||||
|
Command::new("ip2region")
|
||||||
|
.version("0.1")
|
||||||
|
.about("ip2region bin program")
|
||||||
|
.long_about("you can set --db in command to specific the xdb filepath, default run query")
|
||||||
|
.subcommand(Command::new("query").about("query test").arg(&db_arg))
|
||||||
|
.subcommand(
|
||||||
|
Command::new("bench")
|
||||||
|
.about("bench test")
|
||||||
|
.arg(
|
||||||
|
Arg::new("src")
|
||||||
|
.long("src")
|
||||||
|
.help("set this to specific source bench file")
|
||||||
|
.required(true),
|
||||||
|
)
|
||||||
|
.arg(&db_arg),
|
||||||
|
)
|
||||||
|
.get_matches()
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,110 @@
|
||||||
|
extern crate core;
|
||||||
|
|
||||||
|
use std::fs::File;
|
||||||
|
use std::io::Read;
|
||||||
|
use std::io::Write;
|
||||||
|
use std::net::Ipv4Addr;
|
||||||
|
use std::str::FromStr;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use clap::ArgMatches;
|
||||||
|
|
||||||
|
use ip2region2::{search_by_ip, searcher_init};
|
||||||
|
|
||||||
|
mod cmd;
|
||||||
|
|
||||||
|
/// set rust log level, if you don`t want print log, you can skip this
|
||||||
|
fn log_init() {
|
||||||
|
let rust_log_key = "RUST_LOG";
|
||||||
|
std::env::var(rust_log_key).unwrap_or_else(|_| {
|
||||||
|
std::env::set_var(rust_log_key, "INFO");
|
||||||
|
std::env::var(rust_log_key).unwrap()
|
||||||
|
});
|
||||||
|
tracing_subscriber::fmt::init();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bench_test(src_filepath: &str) {
|
||||||
|
let now = Instant::now();
|
||||||
|
let mut count = 0;
|
||||||
|
let mut file = File::open(src_filepath).unwrap();
|
||||||
|
let mut contents = String::new();
|
||||||
|
file.read_to_string(&mut contents).unwrap();
|
||||||
|
|
||||||
|
for line in contents.split('\n') {
|
||||||
|
if !line.contains('|') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let ip_test_line = line.splitn(3, '|').collect::<Vec<&str>>();
|
||||||
|
if ip_test_line.len() != 3 {
|
||||||
|
panic!("this line {line} don`t have enough `|` for spilt");
|
||||||
|
}
|
||||||
|
let start_ip = Ipv4Addr::from_str(ip_test_line[0]).unwrap();
|
||||||
|
let end_ip = Ipv4Addr::from_str(ip_test_line[1]).unwrap();
|
||||||
|
if end_ip < start_ip {
|
||||||
|
panic!("start ip({start_ip}) should not be greater than end ip({end_ip})")
|
||||||
|
}
|
||||||
|
let start_ip = u32::from(start_ip);
|
||||||
|
let end_ip = u32::from(end_ip);
|
||||||
|
let mid_ip = (((start_ip as u64) + (end_ip as u64)) >> 1) as u32;
|
||||||
|
for ip in [
|
||||||
|
start_ip,
|
||||||
|
((start_ip as u64 + mid_ip as u64) >> 1) as u32,
|
||||||
|
mid_ip,
|
||||||
|
((mid_ip as u64 + end_ip as u64) >> 1) as u32,
|
||||||
|
end_ip,
|
||||||
|
] {
|
||||||
|
let result = search_by_ip(ip).unwrap();
|
||||||
|
assert_eq!(result.as_str(), ip_test_line[2]);
|
||||||
|
count += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println!(
|
||||||
|
"Bench finished, total: {count},\
|
||||||
|
took: {:?} ,\
|
||||||
|
cost: {:?}/op",
|
||||||
|
now.elapsed(),
|
||||||
|
now.elapsed() / count
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn query_test() {
|
||||||
|
println!("ip2region xdb searcher test program, type `quit` or `Ctrl + c` to exit");
|
||||||
|
loop {
|
||||||
|
print!("ip2region>> ");
|
||||||
|
std::io::stdout().flush().unwrap();
|
||||||
|
let mut line = String::new();
|
||||||
|
std::io::stdin().read_line(&mut line).unwrap();
|
||||||
|
if line.contains("quit") {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let line = line.trim();
|
||||||
|
let now = Instant::now();
|
||||||
|
let result = search_by_ip(line);
|
||||||
|
let cost = now.elapsed();
|
||||||
|
println!("region: {result:?}, took: {cost:?}",);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn matches_for_searcher(matches: &ArgMatches) {
|
||||||
|
if let Some(xdb_filepath) = matches.get_one::<String>("db") {
|
||||||
|
searcher_init(Some(xdb_filepath.to_owned()))
|
||||||
|
} else {
|
||||||
|
searcher_init(None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
log_init();
|
||||||
|
let matches = cmd::get_matches();
|
||||||
|
if let Some(sub_matches) = matches.subcommand_matches("bench") {
|
||||||
|
matches_for_searcher(sub_matches);
|
||||||
|
let src_filepath = sub_matches.get_one::<String>("src").unwrap();
|
||||||
|
|
||||||
|
bench_test(src_filepath);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(sub_matches) = matches.subcommand_matches("query") {
|
||||||
|
matches_for_searcher(sub_matches);
|
||||||
|
query_test()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
[package]
|
[package]
|
||||||
name = "search"
|
name = "ip2region2"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.66.0"
|
rust-version = "1.66.0"
|
||||||
|
|
@ -10,6 +10,8 @@ license = "Apache-2.0"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
once_cell = "1.16"
|
once_cell = "1.16"
|
||||||
|
tracing = "0.1"
|
||||||
|
tracing-subscriber = "0.3.14"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
criterion = "0.4"
|
criterion = "0.4"
|
||||||
|
|
@ -0,0 +1,52 @@
|
||||||
|
use criterion::{black_box, criterion_group, criterion_main, Criterion};
|
||||||
|
use rand;
|
||||||
|
|
||||||
|
use ip2region2::searcher::{
|
||||||
|
get_block_by_size, get_full_cache, get_vector_index_cache, search_by_ip, searcher_init,
|
||||||
|
};
|
||||||
|
|
||||||
|
fn search_by_ip_bench(c: &mut Criterion) {
|
||||||
|
c.bench_function("search_by_ip_bench", |b| {
|
||||||
|
searcher_init(None);
|
||||||
|
b.iter(|| {
|
||||||
|
search_by_ip(rand::random::<u32>()).unwrap();
|
||||||
|
})
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_block_by_size_bench(c: &mut Criterion) {
|
||||||
|
c.bench_function("get_block_by_size_bench", |b| {
|
||||||
|
b.iter(|| {
|
||||||
|
black_box(get_block_by_size(
|
||||||
|
get_full_cache(),
|
||||||
|
rand::random::<u16>() as usize,
|
||||||
|
4,
|
||||||
|
));
|
||||||
|
})
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_full_cache_bench(c: &mut Criterion) {
|
||||||
|
c.bench_function("get_full_cache_bench", |b| {
|
||||||
|
b.iter(|| {
|
||||||
|
black_box(get_full_cache());
|
||||||
|
})
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_vec_index_cache_bench(c: &mut Criterion) {
|
||||||
|
c.bench_function("get_vec_index_cache_bench", |b| {
|
||||||
|
b.iter(|| {
|
||||||
|
black_box(get_vector_index_cache());
|
||||||
|
})
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
criterion_group!(
|
||||||
|
benches,
|
||||||
|
search_by_ip_bench,
|
||||||
|
get_block_by_size_bench,
|
||||||
|
get_full_cache_bench,
|
||||||
|
get_vec_index_cache_bench,
|
||||||
|
);
|
||||||
|
criterion_main!(benches);
|
||||||
|
|
@ -0,0 +1,4 @@
|
||||||
|
mod ip_value;
|
||||||
|
pub use self::ip_value::ToUIntIP;
|
||||||
|
pub mod searcher;
|
||||||
|
pub use searcher::{search_by_ip, searcher_init};
|
||||||
|
|
@ -0,0 +1,173 @@
|
||||||
|
use std::error::Error;
|
||||||
|
use std::fmt::Display;
|
||||||
|
use std::fs::File;
|
||||||
|
use std::io::Read;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
|
||||||
|
use crate::ToUIntIP;
|
||||||
|
|
||||||
|
const HEADER_INFO_LENGTH: usize = 256;
|
||||||
|
const VECTOR_INDEX_COLS: usize = 256;
|
||||||
|
const VECTOR_INDEX_SIZE: usize = 8;
|
||||||
|
const SEGMENT_INDEX_SIZE: usize = 14;
|
||||||
|
const VECTOR_INDEX_LENGTH: usize = 512 * 1024;
|
||||||
|
|
||||||
|
const XDB_FILEPATH_ENV: &str = "XDB_FILEPATH";
|
||||||
|
|
||||||
|
static CACHE: OnceCell<Vec<u8>> = OnceCell::new();
|
||||||
|
|
||||||
|
/// check https://mp.weixin.qq.com/s/ndjzu0BgaeBmDOCw5aqHUg for details
|
||||||
|
pub fn search_by_ip<T>(ip: T) -> Result<String, Box<dyn Error>>
|
||||||
|
where
|
||||||
|
T: ToUIntIP + Display,
|
||||||
|
{
|
||||||
|
let ip = ip.to_u32_ip()?;
|
||||||
|
let (start_ptr, end_ptr) = get_start_end_ptr(ip);
|
||||||
|
let mut left: usize = 0;
|
||||||
|
let mut right: usize = (end_ptr - start_ptr) / SEGMENT_INDEX_SIZE;
|
||||||
|
|
||||||
|
while left <= right {
|
||||||
|
let mid = (left + right) >> 1;
|
||||||
|
let offset = start_ptr + mid * SEGMENT_INDEX_SIZE;
|
||||||
|
let buffer_ip_value = &get_full_cache()[offset..offset + SEGMENT_INDEX_SIZE];
|
||||||
|
let start_ip = get_block_by_size(buffer_ip_value, 0, 4);
|
||||||
|
if ip < (start_ip as u32) {
|
||||||
|
right = mid - 1;
|
||||||
|
} else if ip > (get_block_by_size(buffer_ip_value, 4, 4) as u32) {
|
||||||
|
left = mid + 1;
|
||||||
|
} else {
|
||||||
|
let data_length = get_block_by_size(buffer_ip_value, 8, 2);
|
||||||
|
let data_offset = get_block_by_size(buffer_ip_value, 10, 4);
|
||||||
|
let result = String::from_utf8(
|
||||||
|
get_full_cache()[data_offset..(data_offset + data_length)].to_vec(),
|
||||||
|
);
|
||||||
|
return Ok(result?);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err("not matched".into())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_start_end_ptr(ip: u32) -> (usize, usize) {
|
||||||
|
let il0 = ((ip >> 24) & 0xFF) as usize;
|
||||||
|
let il1 = ((ip >> 16) & 0xFF) as usize;
|
||||||
|
let idx = VECTOR_INDEX_SIZE * (il0 * VECTOR_INDEX_COLS + il1);
|
||||||
|
let start_point = idx;
|
||||||
|
let vector_cache = get_vector_index_cache();
|
||||||
|
let start_ptr = get_block_by_size(vector_cache, start_point, 4);
|
||||||
|
let end_ptr = get_block_by_size(vector_cache, start_point + 4, 4);
|
||||||
|
(start_ptr, end_ptr)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// it will check ../data/ip2region.xdb, ../../data/ip2region.xdb, ../../../data/ip2region.xdb
|
||||||
|
fn default_detect_xdb_file() -> Result<String, Box<dyn Error>> {
|
||||||
|
let prefix = "../".to_owned();
|
||||||
|
for recurse in 1..4 {
|
||||||
|
let filepath = prefix.repeat(recurse) + "data/ip2region.xdb";
|
||||||
|
if Path::new(filepath.as_str()).exists() {
|
||||||
|
return Ok(filepath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err("default filepath not find the xdb file, so you must set xdb_filepath".into())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub fn get_block_by_size(bytes: &[u8], offset: usize, length: usize) -> usize {
|
||||||
|
let mut result: usize = 0;
|
||||||
|
for (index, value) in bytes[offset..offset + length].iter().enumerate() {
|
||||||
|
result += usize::from(*value) << (index << 3);
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn searcher_init(xdb_filepath: Option<String>) {
|
||||||
|
let xdb_filepath = xdb_filepath.unwrap_or_else(|| default_detect_xdb_file().unwrap());
|
||||||
|
std::env::set_var(XDB_FILEPATH_ENV, xdb_filepath);
|
||||||
|
CACHE.get_or_init(load_file);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_vector_index_cache() -> &'static [u8] {
|
||||||
|
let full_cache: &'static Vec<u8> = get_full_cache();
|
||||||
|
&full_cache[HEADER_INFO_LENGTH..(HEADER_INFO_LENGTH + VECTOR_INDEX_LENGTH)]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_file() -> Vec<u8> {
|
||||||
|
let xdb_filepath =
|
||||||
|
std::env::var("XDB_FILEPATH").unwrap_or_else(|_| default_detect_xdb_file().unwrap());
|
||||||
|
tracing::debug!("load xdb searcher file at {} ", xdb_filepath);
|
||||||
|
let mut f = File::open(xdb_filepath).expect("file open error");
|
||||||
|
let mut buffer = Vec::new();
|
||||||
|
f.read_to_end(&mut buffer).expect("load file error");
|
||||||
|
buffer
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_full_cache() -> &'static Vec<u8> {
|
||||||
|
CACHE.get_or_init(load_file)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::fs::File;
|
||||||
|
use std::io::Read;
|
||||||
|
use std::net::Ipv4Addr;
|
||||||
|
use std::str::FromStr;
|
||||||
|
use std::thread;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
///test all types find correct
|
||||||
|
#[test]
|
||||||
|
fn test_multi_type_ip() {
|
||||||
|
searcher_init(None);
|
||||||
|
|
||||||
|
search_by_ip("2.0.0.0").unwrap();
|
||||||
|
search_by_ip("32").unwrap();
|
||||||
|
search_by_ip(4294408949).unwrap();
|
||||||
|
search_by_ip(Ipv4Addr::from_str("1.1.1.1").unwrap()).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_match_all_ip_correct() {
|
||||||
|
searcher_init(None);
|
||||||
|
let mut file = File::open("../../../data/ip.test.txt").unwrap();
|
||||||
|
let mut contents = String::new();
|
||||||
|
file.read_to_string(&mut contents).unwrap();
|
||||||
|
for line in contents.split("\n") {
|
||||||
|
if !line.contains("|") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let ip_test_line = line.splitn(3, "|").collect::<Vec<&str>>();
|
||||||
|
let start_ip = Ipv4Addr::from_str(ip_test_line[0]).unwrap();
|
||||||
|
let end_ip = Ipv4Addr::from_str(ip_test_line[1]).unwrap();
|
||||||
|
for value in u32::from(start_ip)..u32::from(end_ip) + 1 {
|
||||||
|
let result = search_by_ip(value).unwrap();
|
||||||
|
assert_eq!(result.as_str(), ip_test_line[2])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_multi_thread_only_load_xdb_once() {
|
||||||
|
searcher_init(None);
|
||||||
|
let handle = thread::spawn(|| {
|
||||||
|
let result = search_by_ip("2.2.2.2").unwrap();
|
||||||
|
println!("ip search in spawn: {result}");
|
||||||
|
});
|
||||||
|
let r = search_by_ip("1.1.1.1").unwrap();
|
||||||
|
println!("ip search in main thread: {r}");
|
||||||
|
handle.join().unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_multi_searcher_init() {
|
||||||
|
for _ in 0..5 {
|
||||||
|
thread::spawn(|| {
|
||||||
|
searcher_init(None);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
searcher_init(None);
|
||||||
|
searcher_init(Some(String::from("test")));
|
||||||
|
search_by_ip(123).unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,16 +0,0 @@
|
||||||
use criterion::{criterion_group, criterion_main, Criterion};
|
|
||||||
use rand;
|
|
||||||
|
|
||||||
use search::search_by_ip;
|
|
||||||
|
|
||||||
fn ip_search_benchmark(c: &mut Criterion) {
|
|
||||||
c.bench_function("ip_search_bench", |b| {
|
|
||||||
b.iter(|| {
|
|
||||||
let ip = rand::random::<u32>();
|
|
||||||
search_by_ip(ip).unwrap();
|
|
||||||
})
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
criterion_group!(benches, ip_search_benchmark);
|
|
||||||
criterion_main!(benches);
|
|
||||||
|
|
@ -1,150 +0,0 @@
|
||||||
mod ip_value;
|
|
||||||
|
|
||||||
use std::env;
|
|
||||||
use std::error::Error;
|
|
||||||
use std::fmt;
|
|
||||||
use std::fmt::Formatter;
|
|
||||||
use std::fs::File;
|
|
||||||
use std::io::Read;
|
|
||||||
|
|
||||||
use once_cell::sync::OnceCell;
|
|
||||||
|
|
||||||
use ip_value::ToUIntIP;
|
|
||||||
|
|
||||||
const HEADER_INFO_LENGTH: u32 = 256;
|
|
||||||
// const VECTOR_INDEX_ROWS: u32 = 256;
|
|
||||||
const VECTOR_INDEX_COLS: u32 = 256;
|
|
||||||
const VECTOR_INDEX_SIZE: u32 = 8;
|
|
||||||
const SEGMENT_INDEX_SIZE: usize = 14;
|
|
||||||
|
|
||||||
pub struct Searcher {
|
|
||||||
pub buffer: Vec<u8>,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn global_searcher() -> &'static Searcher {
|
|
||||||
static SEARCHER: OnceCell<Searcher> = OnceCell::new();
|
|
||||||
SEARCHER.get_or_init(|| {
|
|
||||||
let xdp_filepath = env::var("XDB_FILEPATH").expect("you must set XDB_FILEPATH for search");
|
|
||||||
println!("init xdb searcher at {xdp_filepath}");
|
|
||||||
Searcher::new(xdp_filepath.as_str()).unwrap()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
impl fmt::Display for Searcher {
|
|
||||||
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
|
|
||||||
write!(f, "searcher_with_len {}", self.buffer.len())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn search_by_ip<T>(ip: T) -> Result<String, Box<dyn Error>>
|
|
||||||
where
|
|
||||||
T: ToUIntIP,
|
|
||||||
{
|
|
||||||
let changed_value = ip.to_u32_ip()?;
|
|
||||||
search_by_ip_u32(changed_value)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn search_by_ip_u32(ip: u32) -> Result<String, Box<dyn Error>> {
|
|
||||||
let il0 = (ip >> 24) & 0xFF;
|
|
||||||
let il1 = (ip >> 16) & 0xFF;
|
|
||||||
let idx = VECTOR_INDEX_SIZE * (il0 * VECTOR_INDEX_COLS + il1);
|
|
||||||
|
|
||||||
let start_point = (HEADER_INFO_LENGTH + idx) as usize;
|
|
||||||
let buffer = &global_searcher().buffer;
|
|
||||||
let start_ptr = get_u32(buffer, start_point);
|
|
||||||
let end_ptr = get_u32(buffer, start_point + 4);
|
|
||||||
let mut left: usize = 0;
|
|
||||||
let mut right: usize = ((end_ptr - start_ptr) as usize) / SEGMENT_INDEX_SIZE;
|
|
||||||
while left <= right {
|
|
||||||
let mid = (left + right) >> 1;
|
|
||||||
let offset = (start_ptr as usize) + mid * SEGMENT_INDEX_SIZE;
|
|
||||||
let buffer_ip_value = buffer_value(offset, SEGMENT_INDEX_SIZE);
|
|
||||||
let start_ip = get_u32(buffer_ip_value, 0);
|
|
||||||
if ip < start_ip {
|
|
||||||
right = mid - 1;
|
|
||||||
} else if ip > get_u32(buffer_ip_value, 4) {
|
|
||||||
left = mid + 1;
|
|
||||||
} else {
|
|
||||||
let length = (buffer_ip_value[8] as usize & 0x000000FF)
|
|
||||||
| (buffer_ip_value[9] as usize & 0x0000FF00);
|
|
||||||
|
|
||||||
let offset = get_u32(buffer_ip_value, 10);
|
|
||||||
let result = buffer_value(offset as usize, length)
|
|
||||||
.iter()
|
|
||||||
.map(|x| x.to_owned())
|
|
||||||
.collect::<Vec<u8>>();
|
|
||||||
return Ok(String::from_utf8(result)?);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err("not matched".into())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn buffer_value(offset: usize, length: usize) -> &'static [u8] {
|
|
||||||
&global_searcher().buffer[offset..offset + length]
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Searcher {
|
|
||||||
pub fn new(filepath: &str) -> Result<Self, Box<dyn Error>> {
|
|
||||||
let mut f = File::open(filepath)?;
|
|
||||||
let mut buffer = Vec::new();
|
|
||||||
f.read_to_end(&mut buffer)?;
|
|
||||||
Ok(Self { buffer })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_u32(bytes: &[u8], offset: usize) -> u32 {
|
|
||||||
(bytes[offset] as u32) & 0x000000FF
|
|
||||||
| ((bytes[offset + 1] as u32) << 8) & 0x0000FF00
|
|
||||||
| ((bytes[offset + 2] as u32) << 16) & 0x00FF0000
|
|
||||||
| ((bytes[offset + 3] as u32) << 24) & 0xFF000000
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
use std::net::Ipv4Addr;
|
|
||||||
use std::str::FromStr;
|
|
||||||
use std::thread;
|
|
||||||
|
|
||||||
const TEST_IP_FILEPATH: &str = "../../../data/ip.test.txt";
|
|
||||||
|
|
||||||
///test all types find correct
|
|
||||||
#[test]
|
|
||||||
fn test_search_by_ip() {
|
|
||||||
search_by_ip("2.0.0.0").unwrap();
|
|
||||||
search_by_ip("32").unwrap();
|
|
||||||
search_by_ip(32).unwrap();
|
|
||||||
search_by_ip(Ipv4Addr::from_str("1.1.1.1").unwrap()).unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// test find ip correct use the file ip.test.txt in ../../data
|
|
||||||
#[test]
|
|
||||||
fn test_random_choose_ip() {
|
|
||||||
let mut file = File::open(TEST_IP_FILEPATH).unwrap();
|
|
||||||
let mut contents = String::new();
|
|
||||||
file.read_to_string(&mut contents).unwrap();
|
|
||||||
for line in contents.split("\n") {
|
|
||||||
if !line.contains("|") {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let ip_test_line = line.splitn(3, "|").collect::<Vec<&str>>();
|
|
||||||
let start_ip = Ipv4Addr::from_str(ip_test_line[0]).unwrap();
|
|
||||||
let end_ip = Ipv4Addr::from_str(ip_test_line[1]).unwrap();
|
|
||||||
for value in u32::from(start_ip)..u32::from(end_ip) + 1 {
|
|
||||||
let result = search_by_ip(value).unwrap();
|
|
||||||
assert_eq!(result.as_str(), ip_test_line[2])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_multi_thread() {
|
|
||||||
let handle = thread::spawn(|| {
|
|
||||||
let result = search_by_ip("2.2.2.2").unwrap();
|
|
||||||
println!("ip search in spawn: {result}");
|
|
||||||
});
|
|
||||||
let r = search_by_ip("1.1.1.1").unwrap();
|
|
||||||
println!("ip search in main thread: {r}");
|
|
||||||
handle.join().unwrap();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue